@powersync/service-module-mongodb-storage 0.13.2 → 0.15.0

package/src/storage/implementation/db.ts

@@ -10,13 +10,16 @@ import {
   CheckpointEventDocument,
   ClientConnectionDocument,
   CurrentDataDocument,
+  CurrentDataDocumentV3,
   CustomWriteCheckpointDocument,
   IdSequenceDocument,
   InstanceDocument,
   SourceTableDocument,
+  StorageConfig,
   SyncRuleDocument,
   WriteCheckpointDocument
 } from './models.js';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
 
 export interface PowerSyncMongoOptions {
   /**
@@ -27,6 +30,7 @@ export interface PowerSyncMongoOptions {
 
 export class PowerSyncMongo {
   readonly current_data: mongo.Collection<CurrentDataDocument>;
+  readonly v3_current_data: mongo.Collection<CurrentDataDocumentV3>;
   readonly bucket_data: mongo.Collection<BucketDataDocument>;
   readonly bucket_parameters: mongo.Collection<BucketParameterDocument>;
   readonly op_id_sequence: mongo.Collection<IdSequenceDocument>;
@@ -51,7 +55,8 @@ export class PowerSyncMongo {
     });
     this.db = db;
 
-    this.current_data = db.collection<CurrentDataDocument>('current_data');
+    this.current_data = db.collection('current_data');
+    this.v3_current_data = db.collection('v3_current_data');
     this.bucket_data = db.collection('bucket_data');
     this.bucket_parameters = db.collection('bucket_parameters');
     this.op_id_sequence = db.collection('op_id_sequence');
@@ -66,11 +71,16 @@ export class PowerSyncMongo {
     this.connection_report_events = this.db.collection('connection_report_events');
   }
 
+  versioned(storageConfig: StorageConfig) {
+    return new VersionedPowerSyncMongo(this, storageConfig);
+  }
+
   /**
    * Clear all collections.
    */
   async clear() {
     await this.current_data.deleteMany({});
+    await this.v3_current_data.deleteMany({});
     await this.bucket_data.deleteMany({});
     await this.bucket_parameters.deleteMany({});
     await this.op_id_sequence.deleteMany({});
@@ -171,6 +181,124 @@ export class PowerSyncMongo {
       { name: 'dirty_count' }
     );
   }
+
+  async initializeStorageVersion(storageConfig: StorageConfig) {
+    if (storageConfig.softDeleteCurrentData) {
+      // Initialize the v3_current_data collection, which is used for the new storage version.
+      // No-op if this already exists
+      await this.v3_current_data.createIndex(
+        {
+          '_id.g': 1,
+          pending_delete: 1
+        },
+        {
+          partialFilterExpression: { pending_delete: { $exists: true } },
+          name: 'pending_delete'
+        }
+      );
+    }
+  }
+}
+
+/**
+ * This is similar to PowerSyncMongo, but blocks access to certain collections based on the storage version.
+ */
+export class VersionedPowerSyncMongo {
+  readonly client: mongo.MongoClient;
+  readonly db: mongo.Db;
+
+  readonly storageConfig: StorageConfig;
+  #upstream: PowerSyncMongo;
+
+  constructor(upstream: PowerSyncMongo, storageConfig: StorageConfig) {
+    this.#upstream = upstream;
+    this.client = upstream.client;
+    this.db = upstream.db;
+    this.storageConfig = storageConfig;
+  }
+
+  /**
+   * Uses either `current_data` or `v3_current_data` collection based on the storage version.
+   *
+   * Use in places where it does not matter which version is used.
+   */
+  get common_current_data(): mongo.Collection<CurrentDataDocument> {
+    if (this.storageConfig.softDeleteCurrentData) {
+      return this.#upstream.v3_current_data;
+    } else {
+      return this.#upstream.current_data;
+    }
+  }
+
+  get v1_current_data() {
+    if (this.storageConfig.softDeleteCurrentData) {
+      throw new ServiceAssertionError(
+        'current_data collection should not be used when softDeleteCurrentData is enabled'
+      );
+    }
+    return this.#upstream.current_data;
+  }
+
+  get v3_current_data() {
+    if (!this.storageConfig.softDeleteCurrentData) {
+      throw new ServiceAssertionError(
+        'v3_current_data collection should not be used when softDeleteCurrentData is disabled'
+      );
+    }
+    return this.#upstream.v3_current_data;
+  }
+
+  get bucket_data() {
+    return this.#upstream.bucket_data;
+  }
+
+  get bucket_parameters() {
+    return this.#upstream.bucket_parameters;
+  }
+
+  get op_id_sequence() {
+    return this.#upstream.op_id_sequence;
+  }
+
+  get sync_rules() {
+    return this.#upstream.sync_rules;
+  }
+
+  get source_tables() {
+    return this.#upstream.source_tables;
+  }
+
+  get custom_write_checkpoints() {
+    return this.#upstream.custom_write_checkpoints;
+  }
+
+  get write_checkpoints() {
+    return this.#upstream.write_checkpoints;
+  }
+
+  get instance() {
+    return this.#upstream.instance;
+  }
+
+  get locks() {
+    return this.#upstream.locks;
+  }
+
+  get bucket_state() {
+    return this.#upstream.bucket_state;
+  }
+
+  get checkpoint_events() {
+    return this.#upstream.checkpoint_events;
+  }
+
+  get connection_report_events() {
+    return this.#upstream.connection_report_events;
+  }
+
+  notifyCheckpoint() {
+    return this.#upstream.notifyCheckpoint();
+  }
 }
 
 export function createPowerSyncMongo(config: MongoStorageConfig, options?: lib_mongo.MongoConnectionOptions) {

package/src/storage/implementation/models.ts

@@ -1,7 +1,8 @@
-import { InternalOpId, storage } from '@powersync/service-core';
+import { InternalOpId, SerializedSyncPlan, storage } from '@powersync/service-core';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import { event_types } from '@powersync/service-types';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 
 /**
  * Replica id uniquely identifying a row on the source database.
@@ -37,6 +38,19 @@ export interface CurrentDataDocument {
   lookups: bson.Binary[];
 }
 
+export interface CurrentDataDocumentV3 {
+  _id: SourceKey;
+  data: bson.Binary;
+  buckets: CurrentBucket[];
+  lookups: bson.Binary[];
+  /**
+   * If set, this can be deleted, once there is a consistent checkpoint >= pending_delete.
+   *
+   * This must only be set if buckets = [], lookups = [].
+   */
+  pending_delete?: bigint;
+}
+
 export interface CurrentBucket {
   bucket: string;
   table: string;
@@ -199,11 +213,35 @@ export interface SyncRuleDocument {
   last_fatal_error_ts: Date | null;
 
   content: string;
+  serialized_plan?: SerializedSyncPlan | null;
 
   lock?: {
     id: string;
     expires_at: Date;
   } | null;
+
+  storage_version?: number;
+}
+
+export interface StorageConfig extends storage.StorageVersionConfig {
+  /**
+   * When true, bucket_data.checksum is guaranteed to be persisted as a Long.
+   *
+   * When false, it could also have been persisted as an Int32 or Double, in which case it must be converted to
+   * a Long before summing.
+   */
+  longChecksums: boolean;
+}
+
+const LONG_CHECKSUMS_STORAGE_VERSION = 2;
+
+export function getMongoStorageConfig(storageVersion: number): StorageConfig {
+  const baseConfig = storage.STORAGE_VERSION_CONFIG[storageVersion];
+  if (baseConfig == null) {
+    throw new ServiceError(ErrorCode.PSYNC_S1005, `Unsupported storage version ${storageVersion}`);
+  }
+
+  return { ...baseConfig, longChecksums: storageVersion >= LONG_CHECKSUMS_STORAGE_VERSION };
 }
 
 export interface CheckpointEventDocument {

package/src/storage/storage-index.ts

@@ -2,7 +2,6 @@ export * from './implementation/db.js';
 export * from './implementation/models.js';
 export * from './implementation/MongoBucketBatch.js';
 export * from './implementation/MongoIdSequence.js';
-export * from './implementation/MongoPersistedSyncRules.js';
 export * from './implementation/MongoPersistedSyncRulesContent.js';
 export * from './implementation/MongoStorageProvider.js';
 export * from './implementation/MongoSyncBucketStorage.js';

package/src/utils/test-utils.ts

@@ -1,33 +1,35 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { PowerSyncMongo } from '../storage/implementation/db.js';
 import { TestStorageOptions } from '@powersync/service-core';
+import { MongoBucketStorage, MongoBucketStorageOptions } from '../storage/MongoBucketStorage.js';
 import { MongoReportStorage } from '../storage/MongoReportStorage.js';
-import { MongoBucketStorage } from '../storage/MongoBucketStorage.js';
-import { MongoSyncBucketStorageOptions } from '../storage/implementation/MongoSyncBucketStorage.js';
+import { PowerSyncMongo } from '../storage/implementation/db.js';
 
 export type MongoTestStorageOptions = {
   url: string;
   isCI: boolean;
-  internalOptions?: MongoSyncBucketStorageOptions;
+  internalOptions?: MongoBucketStorageOptions;
 };
 
 export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) {
-  return async (options?: TestStorageOptions) => {
-    const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI);
+  return {
+    factory: async (options?: TestStorageOptions) => {
+      const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI);
 
-    // None of the tests insert data into this collection, so it was never created
-    if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
-      await db.db.createCollection('bucket_parameters');
-    }
+      // None of the tests insert data into this collection, so it was never created
+      if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
+        await db.db.createCollection('bucket_parameters');
+      }
 
-    // Full migrations are not currently run for tests, so we manually create this
-    await db.createCheckpointEventsCollection();
+      // Full migrations are not currently run for tests, so we manually create this
+      await db.createCheckpointEventsCollection();
 
-    if (!options?.doNotClear) {
-      await db.clear();
-    }
+      if (!options?.doNotClear) {
+        await db.clear();
+      }
 
-    return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }, factoryOptions.internalOptions);
+      return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }, factoryOptions.internalOptions);
+    },
+    tableIdStrings: false
   };
 }
 

package/src/utils/util.ts

@@ -91,10 +91,10 @@ export function mapOpEntry(row: BucketDataDocument): utils.OplogEntry {
   }
 }
 
-export function replicaIdToSubkey(table: bson.ObjectId, id: storage.ReplicaId): string {
+export function replicaIdToSubkey(table: storage.SourceTableId, id: storage.ReplicaId): string {
   if (storage.isUUID(id)) {
     // Special case for UUID for backwards-compatiblity
-    return `${table.toHexString()}/${id.toHexString()}`;
+    return `${tableIdString(table)}/${id.toHexString()}`;
   } else {
     // Hashed UUID from the table and id
     const repr = bson.serialize({ table, id });
@@ -102,6 +102,21 @@ export function replicaIdToSubkey(table: bson.ObjectId, id: storage.ReplicaId):
   }
 }
 
+export function mongoTableId(table: storage.SourceTableId): bson.ObjectId {
+  if (typeof table == 'string') {
+    throw new ServiceAssertionError(`Got string table id, expected ObjectId`);
+  }
+  return table;
+}
+
+function tableIdString(table: storage.SourceTableId) {
+  if (typeof table == 'string') {
+    return table;
+  } else {
+    return table.toHexString();
+  }
+}
+
 export function setSessionSnapshotTime(session: mongo.ClientSession, time: bson.Timestamp) {
   // This is a workaround for the lack of direct support for snapshot reads in the MongoDB driver.
   if (!session.snapshotEnabled) {

package/test/src/__snapshots__/storage.test.ts.snap

@@ -1,25 +1,201 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
-exports[`Mongo Sync Bucket Storage - Data > empty storage metrics 1`] = `
-{
-  "operations_size_bytes": 0,
-  "parameters_size_bytes": 0,
-  "replication_size_bytes": 0,
-}
-`;
-
-exports[`Mongo Sync Bucket Storage - split buckets > empty storage metrics 1`] = `
-{
-  "operations_size_bytes": 0,
-  "parameters_size_bytes": 0,
-  "replication_size_bytes": 0,
-}
-`;
-
-exports[`Mongo Sync Bucket Storage - split operations > empty storage metrics 1`] = `
-{
-  "operations_size_bytes": 0,
-  "parameters_size_bytes": 0,
-  "replication_size_bytes": 0,
-}
+exports[`Mongo Sync Bucket Storage - Data > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - Data > (insert, delete, insert), (delete) 2`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - Data > (insert, delete, insert), (delete) 3`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - Data - v1 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - Data - v2 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - Data - v3 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - split buckets > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - split operations > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
 `;

package/test/src/__snapshots__/storage_compacting.test.ts.snap

@@ -0,0 +1,17 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Mongo Sync Bucket Storage Compact > partial checksums after compacting (2) 1`] = `
+{
+  "bucket": "1#global[]",
+  "checksum": -1481659821,
+  "count": 1,
+}
+`;
+
+exports[`Mongo Sync Bucket Storage Compact > partial checksums after compacting 1`] = `
+{
+  "bucket": "1#global[]",
+  "checksum": 1874612650,
+  "count": 4,
+}
+`;