@powersync/service-module-mongodb-storage 0.0.0-dev-20250813080357 → 0.0.0-dev-20250819134004

package/src/storage/MongoReportStorage.ts

@@ -4,7 +4,7 @@ import { event_types } from '@powersync/service-types';
 import { PowerSyncMongo } from './implementation/db.js';
 import { logger } from '@powersync/lib-services-framework';
 
-export class MongoReportStorage implements storage.ReportStorageFactory {
+export class MongoReportStorage implements storage.ReportStorage {
   private readonly client: mongo.MongoClient;
   public readonly db: PowerSyncMongo;
 
@@ -12,6 +12,94 @@ export class MongoReportStorage implements storage.ReportStorageFactory {
     this.client = db.client;
     this.db = db;
   }
+  async deleteOldConnectionData(data: event_types.DeleteOldConnectionData): Promise<void> {
+    const { date } = data;
+    const result = await this.db.sdk_report_events.deleteMany({
+      connected_at: { $lt: date },
+      $or: [
+        { disconnected_at: { $exists: true } },
+        { jwt_exp: { $lt: new Date() }, disconnected_at: { $exists: false } }
+      ]
+    });
+    if (result.deletedCount > 0) {
+      logger.info(
+        `TTL from ${date.toISOString()}: ${result.deletedCount} MongoDB documents have been removed from sdk_report_events.`
+      );
+    }
+  }
+
+  async getClientConnectionReports(
+    data: event_types.ClientConnectionReportRequest
+  ): Promise<event_types.ClientConnectionReport> {
+    const { start, end } = data;
+    const result = await this.db.sdk_report_events
+      .aggregate<event_types.ClientConnectionReport>([
+        {
+          $match: {
+            connected_at: { $lte: end, $gte: start }
+          }
+        },
+        this.connectionsFacetPipeline(),
+        this.connectionsProjectPipeline()
+      ])
+      .toArray();
+    return result[0];
+  }
+
+  async reportClientConnection(data: event_types.ClientConnectionBucketData): Promise<void> {
+    const updateFilter = this.updateDocFilter(data.user_id, data.client_id!);
+    await this.db.sdk_report_events.findOneAndUpdate(
+      updateFilter,
+      {
+        $set: data,
+        $unset: {
+          disconnected_at: ''
+        }
+      },
+      {
+        upsert: true
+      }
+    );
+  }
+  async reportClientDisconnection(data: event_types.ClientDisconnectionEventData): Promise<void> {
+    const { connected_at, user_id, client_id } = data;
+    await this.db.sdk_report_events.findOneAndUpdate(
+      {
+        client_id,
+        user_id,
+        connected_at
+      },
+      {
+        $set: {
+          disconnected_at: data.disconnected_at
+        },
+        $unset: {
+          jwt_exp: ''
+        }
+      }
+    );
+  }
+  async getConnectedClients(data: event_types.ClientConnectionsRequest): Promise<event_types.ClientConnectionReport> {
+    const timeframeFilter = this.listConnectionsDateRange(data);
+    const result = await this.db.sdk_report_events
+      .aggregate<event_types.ClientConnectionReport>([
+        {
+          $match: {
+            disconnected_at: { $exists: false },
+            jwt_exp: { $gt: new Date() },
+            ...timeframeFilter
+          }
+        },
+        this.connectionsFacetPipeline(),
+        this.connectionsProjectPipeline()
+      ])
+      .toArray();
+    return result[0];
+  }
+
+  async [Symbol.asyncDispose]() {
+    // No-op
+  }
 
   private parseJsDate(date: Date) {
     const year = date.getFullYear();
@@ -27,7 +115,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory {
     };
   }
 
-  private sdkFacetPipeline() {
+  private connectionsFacetPipeline() {
     return {
       $facet: {
         unique_users: [
@@ -67,7 +155,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory {
     };
   }
 
-  private sdkProjectPipeline() {
+  private connectionsProjectPipeline() {
     return {
       $project: {
         users: { $ifNull: [{ $arrayElemAt: ['$unique_users.count', 0] }, 0] },
@@ -82,7 +170,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory {
     return {
       user_id: userId,
       client_id: clientId,
-      connect_at: {
+      connected_at: {
         // Need to create a new date here to sett the time to 00:00:00
         $gte: new Date(year, month, today),
         $lt: new Date(year, month, nextDay)
@@ -90,7 +178,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory {
     };
   }
 
-  private listConnectionsDateRange(data: event_types.ListCurrentConnectionsRequest) {
+  private listConnectionsDateRange(data: event_types.ClientConnectionsRequest) {
     const { range } = data;
     if (!range) {
       return undefined;
@@ -98,96 +186,10 @@ export class MongoReportStorage implements storage.ReportStorageFactory {
     const endDate = data.range?.end ? new Date(data.range.end) : new Date();
     const startDate = new Date(range.start);
     return {
-      connect_at: {
+      connected_at: {
         $lte: endDate,
         $gte: startDate
       }
     };
   }
-
-  async deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise<void> {
-    const { date } = data;
-    const result = await this.db.sdk_report_events.deleteMany({
-      connect_at: { $lt: date },
-      $or: [{ disconnect_at: { $exists: true } }, { jwt_exp: { $lt: new Date() }, disconnect_at: { $exists: false } }]
-    });
-    if (result.deletedCount > 0) {
-      logger.info(
-        `TTL from ${date.toISOString()}: ${result.deletedCount} MongoDB documents have been removed from sdk_report_events.`
-      );
-    }
-  }
-
-  async scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise<event_types.ListCurrentConnections> {
-    const { start, end } = data;
-    const result = await this.db.sdk_report_events
-      .aggregate<event_types.ListCurrentConnections>([
-        {
-          $match: {
-            connect_at: { $lte: end, $gte: start }
-          }
-        },
-        this.sdkFacetPipeline(),
-        this.sdkProjectPipeline()
-      ])
-      .toArray();
-    return result[0];
-  }
-
-  async reportSdkConnect(data: event_types.SdkConnectBucketData): Promise<void> {
-    const updateFilter = this.updateDocFilter(data.user_id, data.client_id!);
-    await this.db.sdk_report_events.findOneAndUpdate(
-      updateFilter,
-      {
-        $set: data,
-        $unset: {
-          disconnect_at: ''
-        }
-      },
-      {
-        upsert: true
-      }
-    );
-  }
-  async reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise<void> {
-    const { connect_at, user_id, client_id } = data;
-    await this.db.sdk_report_events.findOneAndUpdate(
-      {
-        client_id,
-        user_id,
-        connect_at
-      },
-      {
-        $set: {
-          disconnect_at: data.disconnect_at
-        },
-        $unset: {
-          jwt_exp: ''
-        }
-      }
-    );
-  }
-  async listCurrentConnections(
-    data: event_types.ListCurrentConnectionsRequest
-  ): Promise<event_types.ListCurrentConnections> {
-    const timeframeFilter = this.listConnectionsDateRange(data);
-    const result = await this.db.sdk_report_events
-      .aggregate<event_types.ListCurrentConnections>([
-        {
-          $match: {
-            disconnect_at: { $exists: false },
-            jwt_exp: { $gt: new Date() },
-            ...timeframeFilter
-          }
-        },
-        this.sdkFacetPipeline(),
-        this.sdkProjectPipeline()
-      ])
-      .toArray();
-    return result[0];
-  }
-
-  async [Symbol.asyncDispose]() {
-    // No-op
-  }
 }

package/src/storage/implementation/MongoBucketBatch.ts

@@ -807,7 +807,7 @@ export class MongoBucketBatch
   }
 
   async keepalive(lsn: string): Promise<boolean> {
-    if (this.last_checkpoint_lsn != null && lsn <= this.last_checkpoint_lsn) {
+    if (this.last_checkpoint_lsn != null && lsn < this.last_checkpoint_lsn) {
       // No-op
       return false;
     }

package/src/storage/implementation/MongoStorageProvider.ts

@@ -6,7 +6,7 @@ import { MongoBucketStorage } from '../MongoBucketStorage.js';
 import { PowerSyncMongo } from './db.js';
 import { MongoReportStorage } from '../MongoReportStorage.js';
 
-export class MongoStorageProvider implements storage.BucketStorageProvider {
+export class MongoStorageProvider implements storage.StorageProvider {
   get type() {
     return lib_mongo.MONGO_CONNECTION_TYPE;
   }

package/src/storage/implementation/db.ts

@@ -8,11 +8,11 @@ import {
   BucketParameterDocument,
   BucketStateDocument,
   CheckpointEventDocument,
+  ClientConnectionDocument,
   CurrentDataDocument,
   CustomWriteCheckpointDocument,
   IdSequenceDocument,
   InstanceDocument,
-  SdkConnectDocument,
   SourceTableDocument,
   SyncRuleDocument,
   WriteCheckpointDocument
@@ -38,7 +38,7 @@ export class PowerSyncMongo {
   readonly locks: mongo.Collection<lib_mongo.locks.Lock>;
   readonly bucket_state: mongo.Collection<BucketStateDocument>;
   readonly checkpoint_events: mongo.Collection<CheckpointEventDocument>;
-  readonly sdk_report_events: mongo.Collection<SdkConnectDocument>;
+  readonly sdk_report_events: mongo.Collection<ClientConnectionDocument>;
 
   readonly client: mongo.MongoClient;
   readonly db: mongo.Db;

package/src/storage/implementation/models.ts

@@ -221,4 +221,4 @@ export interface InstanceDocument {
   _id: string;
 }
 
-export interface SdkConnectDocument extends event_types.SdkConnectDocument {}
+export interface ClientConnectionDocument extends event_types.ClientConnection {}

package/test/src/__snapshots__/storage_sync.test.ts.snap

@@ -10,9 +10,21 @@ exports[`sync - mongodb > compacting data - invalidate checkpoint 1`] = `
           "checksum": -93886621,
           "count": 2,
           "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
         },
       ],
       "last_op_id": "2",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "mybucket",
+        },
+      ],
       "write_checkpoint": undefined,
     },
   },
@@ -46,6 +58,11 @@ exports[`sync - mongodb > compacting data - invalidate checkpoint 2`] = `
           "checksum": 499012468,
           "count": 4,
           "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
         },
       ],
       "write_checkpoint": undefined,
@@ -105,9 +122,21 @@ exports[`sync - mongodb > expiring token 1`] = `
           "checksum": 0,
           "count": 0,
           "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
         },
       ],
       "last_op_id": "0",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "mybucket",
+        },
+      ],
       "write_checkpoint": undefined,
     },
   },
@@ -137,9 +166,21 @@ exports[`sync - mongodb > sends checkpoint complete line for empty checkpoint 1`
           "checksum": -1221282404,
           "count": 1,
           "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
         },
       ],
       "last_op_id": "1",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "mybucket",
+        },
+      ],
       "write_checkpoint": undefined,
     },
   },
@@ -193,15 +234,37 @@ exports[`sync - mongodb > sync buckets in order 1`] = `
           "checksum": 920318466,
           "count": 1,
           "priority": 2,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
         },
         {
           "bucket": "b1[]",
           "checksum": -1382098757,
           "count": 1,
           "priority": 1,
+          "subscriptions": [
+            {
+              "default": 1,
+            },
+          ],
         },
       ],
       "last_op_id": "2",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "b0",
+        },
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "b1",
+        },
+      ],
       "write_checkpoint": undefined,
     },
   },
@@ -267,9 +330,21 @@ exports[`sync - mongodb > sync global data 1`] = `
           "checksum": -93886621,
           "count": 2,
           "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
         },
       ],
       "last_op_id": "2",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "mybucket",
+        },
+      ],
       "write_checkpoint": undefined,
     },
   },
@@ -319,21 +394,53 @@ exports[`sync - mongodb > sync interrupts low-priority buckets on new checkpoint
           "checksum": -659831575,
           "count": 2000,
           "priority": 2,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
         },
         {
           "bucket": "b0b[]",
           "checksum": -659831575,
           "count": 2000,
           "priority": 2,
+          "subscriptions": [
+            {
+              "default": 1,
+            },
+          ],
         },
         {
           "bucket": "b1[]",
           "checksum": -1096116670,
           "count": 1,
           "priority": 1,
+          "subscriptions": [
+            {
+              "default": 2,
+            },
+          ],
         },
       ],
       "last_op_id": "4001",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "b0a",
+        },
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "b0b",
+        },
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "b1",
+        },
+      ],
       "write_checkpoint": undefined,
     },
   },
@@ -380,18 +487,33 @@ exports[`sync - mongodb > sync interrupts low-priority buckets on new checkpoint
           "checksum": 883076828,
           "count": 2001,
           "priority": 2,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
         },
         {
           "bucket": "b0b[]",
           "checksum": 883076828,
           "count": 2001,
           "priority": 2,
+          "subscriptions": [
+            {
+              "default": 1,
+            },
+          ],
         },
         {
           "bucket": "b1[]",
           "checksum": 1841937527,
           "count": 2,
           "priority": 1,
+          "subscriptions": [
+            {
+              "default": 2,
+            },
+          ],
         },
       ],
       "write_checkpoint": undefined,
@@ -466,9 +588,21 @@ exports[`sync - mongodb > sync legacy non-raw data 1`] = `
           "checksum": -852817836,
           "count": 1,
           "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
         },
       ],
       "last_op_id": "1",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "mybucket",
+        },
+      ],
       "write_checkpoint": undefined,
     },
   },
@@ -514,9 +648,21 @@ exports[`sync - mongodb > sync updates to data query only 1`] = `
           "checksum": 0,
           "count": 0,
           "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
         },
       ],
       "last_op_id": "1",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "by_user",
+        },
+      ],
       "write_checkpoint": undefined,
     },
   },
@@ -540,6 +686,11 @@ exports[`sync - mongodb > sync updates to data query only 2`] = `
           "checksum": 1418351250,
           "count": 1,
           "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
         },
       ],
       "write_checkpoint": undefined,
@@ -582,9 +733,21 @@ exports[`sync - mongodb > sync updates to global data 1`] = `
           "checksum": 0,
           "count": 0,
           "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
         },
       ],
       "last_op_id": "0",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "mybucket",
+        },
+      ],
       "write_checkpoint": undefined,
     },
   },
@@ -608,6 +771,11 @@ exports[`sync - mongodb > sync updates to global data 2`] = `
           "checksum": 920318466,
           "count": 1,
           "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
         },
       ],
       "write_checkpoint": undefined,
@@ -652,6 +820,11 @@ exports[`sync - mongodb > sync updates to global data 3`] = `
           "checksum": -93886621,
           "count": 2,
           "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
         },
       ],
       "write_checkpoint": undefined,
@@ -690,6 +863,13 @@ exports[`sync - mongodb > sync updates to parameter query + data 1`] = `
     "checkpoint": {
       "buckets": [],
       "last_op_id": "0",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "by_user",
+        },
+      ],
       "write_checkpoint": undefined,
     },
   },
@@ -713,6 +893,11 @@ exports[`sync - mongodb > sync updates to parameter query + data 2`] = `
           "checksum": 1418351250,
           "count": 1,
           "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
         },
       ],
       "write_checkpoint": undefined,
@@ -751,6 +936,13 @@ exports[`sync - mongodb > sync updates to parameter query only 1`] = `
     "checkpoint": {
       "buckets": [],
       "last_op_id": "0",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "by_user",
+        },
+      ],
       "write_checkpoint": undefined,
     },
   },
@@ -774,6 +966,11 @@ exports[`sync - mongodb > sync updates to parameter query only 2`] = `
           "checksum": 0,
           "count": 0,
           "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
         },
       ],
       "write_checkpoint": undefined,