@powersync/service-module-mongodb-storage 0.0.0-dev-20260225160713 → 0.0.0-dev-20260313100403

package/src/storage/implementation/models.ts

@@ -2,6 +2,7 @@ import { InternalOpId, SerializedSyncPlan, storage } from '@powersync/service-co
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import { event_types } from '@powersync/service-types';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 
 /**
  * Replica id uniquely identifying a row on the source database.
@@ -37,6 +38,19 @@ export interface CurrentDataDocument {
   lookups: bson.Binary[];
 }
 
+export interface CurrentDataDocumentV3 {
+  _id: SourceKey;
+  data: bson.Binary;
+  buckets: CurrentBucket[];
+  lookups: bson.Binary[];
+  /**
+   * If set, this can be deleted, once there is a consistent checkpoint >= pending_delete.
+   *
+   * This must only be set if buckets = [], lookups = [].
+   */
+  pending_delete?: bigint;
+}
+
 export interface CurrentBucket {
   bucket: string;
   table: string;
@@ -221,10 +235,10 @@ export interface StorageConfig extends storage.StorageVersionConfig {
 
 const LONG_CHECKSUMS_STORAGE_VERSION = 2;
 
-export function getMongoStorageConfig(storageVersion: number): StorageConfig | undefined {
+export function getMongoStorageConfig(storageVersion: number): StorageConfig {
   const baseConfig = storage.STORAGE_VERSION_CONFIG[storageVersion];
   if (baseConfig == null) {
-    return undefined;
+    throw new ServiceError(ErrorCode.PSYNC_S1005, `Unsupported storage version ${storageVersion}`);
   }
 
   return { ...baseConfig, longChecksums: storageVersion >= LONG_CHECKSUMS_STORAGE_VERSION };

package/src/utils/test-utils.ts

@@ -11,22 +11,25 @@ export type MongoTestStorageOptions = {
 };
 
 export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) {
-  return async (options?: TestStorageOptions) => {
-    const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI);
+  return {
+    factory: async (options?: TestStorageOptions) => {
+      const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI);
 
-    // None of the tests insert data into this collection, so it was never created
-    if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
-      await db.db.createCollection('bucket_parameters');
-    }
+      // None of the tests insert data into this collection, so it was never created
+      if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
+        await db.db.createCollection('bucket_parameters');
+      }
 
-    // Full migrations are not currently run for tests, so we manually create this
-    await db.createCheckpointEventsCollection();
+      // Full migrations are not currently run for tests, so we manually create this
+      await db.createCheckpointEventsCollection();
 
-    if (!options?.doNotClear) {
-      await db.clear();
-    }
+      if (!options?.doNotClear) {
+        await db.clear();
+      }
 
-    return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }, factoryOptions.internalOptions);
+      return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }, factoryOptions.internalOptions);
+    },
+    tableIdStrings: false
   };
 }
 

package/src/utils/util.ts

@@ -91,10 +91,10 @@ export function mapOpEntry(row: BucketDataDocument): utils.OplogEntry {
   }
 }
 
-export function replicaIdToSubkey(table: bson.ObjectId, id: storage.ReplicaId): string {
+export function replicaIdToSubkey(table: storage.SourceTableId, id: storage.ReplicaId): string {
   if (storage.isUUID(id)) {
     // Special case for UUID for backwards-compatiblity
-    return `${table.toHexString()}/${id.toHexString()}`;
+    return `${tableIdString(table)}/${id.toHexString()}`;
   } else {
     // Hashed UUID from the table and id
     const repr = bson.serialize({ table, id });
@@ -102,6 +102,21 @@ export function replicaIdToSubkey(table: bson.ObjectId, id: storage.ReplicaId):
   }
 }
 
+export function mongoTableId(table: storage.SourceTableId): bson.ObjectId {
+  if (typeof table == 'string') {
+    throw new ServiceAssertionError(`Got string table id, expected ObjectId`);
+  }
+  return table;
+}
+
+function tableIdString(table: storage.SourceTableId) {
+  if (typeof table == 'string') {
+    return table;
+  } else {
+    return table.toHexString();
+  }
+}
+
 export function setSessionSnapshotTime(session: mongo.ClientSession, time: bson.Timestamp) {
   // This is a workaround for the lack of direct support for snapshot reads in the MongoDB driver.
   if (!session.snapshotEnabled) {

package/test/src/__snapshots__/{connection-report-storage.test.ts.snap → client-connections-storage.test.ts.snap}

@@ -19,7 +19,7 @@ exports[`Connection reporting storage > Should create a connection report if its
 
 exports[`Connection reporting storage > Should delete rows older than specified range 1`] = `
 {
-  "sdks": [
+  "sdk_breakdown": [
     {
       "clients": 1,
       "sdk": "powersync-dart/1.6.4",
@@ -51,7 +51,7 @@ exports[`Connection reporting storage > Should delete rows older than specified
       "users": 1,
     },
   ],
-  "users": 5,
+  "total_users": 5,
 }
 `;
 
@@ -77,9 +77,9 @@ exports[`Connection reporting storage > Should update a connection report if its
 ]
 `;
 
-exports[`Report storage tests > Should show connection report data for user over the past day 1`] = `
+exports[`Report storage tests > getClientConnectionsSummary returns the past day summary 1`] = `
 {
-  "sdks": [
+  "sdk_breakdown": [
     {
       "clients": 1,
       "sdk": "powersync-dart/1.6.4",
@@ -101,13 +101,13 @@ exports[`Report storage tests > Should show connection report data for user over
       "users": 1,
     },
   ],
-  "users": 3,
+  "total_users": 3,
 }
 `;
 
-exports[`Report storage tests > Should show connection report data for user over the past month 1`] = `
+exports[`Report storage tests > getClientConnectionsSummary returns the past month summary 1`] = `
 {
-  "sdks": [
+  "sdk_breakdown": [
     {
       "clients": 1,
       "sdk": "powersync-dart/1.6.4",
@@ -149,13 +149,13 @@ exports[`Report storage tests > Should show connection report data for user over
       "users": 1,
     },
   ],
-  "users": 7,
+  "total_users": 7,
 }
 `;
 
-exports[`Report storage tests > Should show connection report data for user over the past week 1`] = `
+exports[`Report storage tests > getClientConnectionsSummary returns the past week summary 1`] = `
 {
-  "sdks": [
+  "sdk_breakdown": [
     {
       "clients": 1,
       "sdk": "powersync-dart/1.6.4",
@@ -187,50 +187,11 @@ exports[`Report storage tests > Should show connection report data for user over
       "users": 1,
     },
   ],
-  "users": 5,
+  "total_users": 5,
 }
 `;
 
-exports[`Report storage tests > Should show currently connected users 1`] = `
-{
-  "sdks": [
-    {
-      "clients": 1,
-      "sdk": "powersync-dart/1.6.4",
-      "users": 1,
-    },
-    {
-      "clients": 1,
-      "sdk": "powersync-js/1.21.1",
-      "users": 1,
-    },
-    {
-      "clients": 1,
-      "sdk": "unknown",
-      "users": 1,
-    },
-  ],
-  "users": 2,
-}
-`;
-
-exports[`Report storage tests > Should show paginated response of all connections of specified client_id 1`] = `
-{
-  "count": 1,
-  "cursor": undefined,
-  "items": [
-    {
-      "client_id": "client_two",
-      "sdk": "powersync-js/1.21.1",
-      "user_agent": "powersync-js/1.21.0 powersync-web Chromium/138 linux",
-      "user_id": "user_two",
-    },
-  ],
-  "more": false,
-}
-`;
-
-exports[`Report storage tests > Should show paginated response of all connections with a limit 1`] = `
+exports[`Report storage tests > getClientSessions returns paginated sessions with a limit 1`] = `
 {
   "count": 4,
   "cursor": "<removed-for-snapshot>",
@@ -264,7 +225,7 @@ exports[`Report storage tests > Should show paginated response of all connection
 }
 `;
 
-exports[`Report storage tests > Should show paginated response of all connections with a limit 2`] = `
+exports[`Report storage tests > getClientSessions returns paginated sessions with a limit 2`] = `
 {
   "count": 4,
   "cursor": undefined,
@@ -298,7 +259,7 @@ exports[`Report storage tests > Should show paginated response of all connection
 }
 `;
 
-exports[`Report storage tests > Should show paginated response of all connections with a limit with date range 1`] = `
+exports[`Report storage tests > getClientSessions returns paginated sessions with a limit and date range 1`] = `
 {
   "count": 4,
   "cursor": "<removed-for-snapshot>",
@@ -332,7 +293,7 @@ exports[`Report storage tests > Should show paginated response of all connection
 }
 `;
 
-exports[`Report storage tests > Should show paginated response of all connections with a limit with date range 2`] = `
+exports[`Report storage tests > getClientSessions returns paginated sessions with a limit and date range 2`] = `
 {
   "count": 2,
   "cursor": undefined,
@@ -366,29 +327,23 @@ exports[`Report storage tests > Should show paginated response of all connection
 }
 `;
 
-exports[`Report storage tests > Should show paginated response of connections of specified user_id 1`] = `
+exports[`Report storage tests > getClientSessions returns sessions for a client_id 1`] = `
 {
-  "count": 2,
+  "count": 1,
   "cursor": undefined,
   "items": [
     {
-      "client_id": "client_one",
-      "sdk": "powersync-dart/1.6.4",
-      "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android",
-      "user_id": "user_one",
-    },
-    {
-      "client_id": "",
-      "sdk": "unknown",
-      "user_agent": "Dart (flutter-web) Chrome/128 android",
-      "user_id": "user_one",
+      "client_id": "client_two",
+      "sdk": "powersync-js/1.21.1",
+      "user_agent": "powersync-js/1.21.0 powersync-web Chromium/138 linux",
+      "user_id": "user_two",
     },
   ],
   "more": false,
 }
 `;
 
-exports[`Report storage tests > Should show paginated response of connections over a date range 1`] = `
+exports[`Report storage tests > getClientSessions returns sessions for a date range 1`] = `
 {
   "count": 6,
   "cursor": undefined,
@@ -434,7 +389,29 @@ exports[`Report storage tests > Should show paginated response of connections ov
 }
 `;
 
-exports[`Report storage tests > Should show paginated response of connections over a date range of specified client_id and user_id 1`] = `
+exports[`Report storage tests > getClientSessions returns sessions for a user_id 1`] = `
+{
+  "count": 2,
+  "cursor": undefined,
+  "items": [
+    {
+      "client_id": "client_one",
+      "sdk": "powersync-dart/1.6.4",
+      "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android",
+      "user_id": "user_one",
+    },
+    {
+      "client_id": "",
+      "sdk": "unknown",
+      "user_agent": "Dart (flutter-web) Chrome/128 android",
+      "user_id": "user_one",
+    },
+  ],
+  "more": false,
+}
+`;
+
+exports[`Report storage tests > getClientSessions returns sessions for client_id and user_id within a date range 1`] = `
 {
   "count": 1,
   "cursor": undefined,
@@ -449,3 +426,26 @@ exports[`Report storage tests > Should show paginated response of connections ov
   "more": false,
 }
 `;
+
+exports[`Report storage tests > getCurrentConnections returns current connections 1`] = `
+{
+  "sdk_breakdown": [
+    {
+      "clients": 1,
+      "sdk": "powersync-dart/1.6.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.1",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "unknown",
+      "users": 1,
+    },
+  ],
+  "total_users": 2,
+}
+`;

package/test/src/__snapshots__/storage.test.ts.snap

@@ -0,0 +1,201 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Mongo Sync Bucket Storage - Data > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - Data > (insert, delete, insert), (delete) 2`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - Data > (insert, delete, insert), (delete) 3`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - Data - v1 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - Data - v2 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - Data - v3 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - split buckets > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Mongo Sync Bucket Storage - split operations > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;

package/test/src/__snapshots__/storage_compacting.test.ts.snap

@@ -0,0 +1,17 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Mongo Sync Bucket Storage Compact > partial checksums after compacting (2) 1`] = `
+{
+  "bucket": "1#global[]",
+  "checksum": -1481659821,
+  "count": 1,
+}
+`;
+
+exports[`Mongo Sync Bucket Storage Compact > partial checksums after compacting 1`] = `
+{
+  "bucket": "1#global[]",
+  "checksum": 1874612650,
+  "count": 4,
+}
+`;