@powersync/service-module-mongodb-storage 0.12.8 → 0.12.9

package/src/storage/MongoReportStorage.ts

@@ -0,0 +1,174 @@
+import { storage } from '@powersync/service-core';
+import { event_types } from '@powersync/service-types';
+import { PowerSyncMongo } from './implementation/db.js';
+import { logger } from '@powersync/lib-services-framework';
+
+export class MongoReportStorage implements storage.ReportStorage {
+  public readonly db: PowerSyncMongo;
+
+  constructor(db: PowerSyncMongo) {
+    this.db = db;
+  }
+  async deleteOldConnectionData(data: event_types.DeleteOldConnectionData): Promise<void> {
+    const { date } = data;
+    const result = await this.db.connection_report_events.deleteMany({
+      connected_at: { $lt: date },
+      $or: [
+        { disconnected_at: { $exists: true } },
+        { jwt_exp: { $lt: new Date() }, disconnected_at: { $exists: false } }
+      ]
+    });
+    if (result.deletedCount > 0) {
+      logger.info(
+        `TTL from ${date.toISOString()}: ${result.deletedCount} MongoDB documents have been removed from connection_report_events.`
+      );
+    }
+  }
+
+  async getClientConnectionReports(
+    data: event_types.ClientConnectionReportRequest
+  ): Promise<event_types.ClientConnectionReportResponse> {
+    const { start, end } = data;
+    const result = await this.db.connection_report_events
+      .aggregate<event_types.ClientConnectionReportResponse>([
+        {
+          $match: {
+            connected_at: { $lte: end, $gte: start }
+          }
+        },
+        this.connectionsFacetPipeline(),
+        this.connectionsProjectPipeline()
+      ])
+      .toArray();
+    return result[0];
+  }
+
+  async reportClientConnection(data: event_types.ClientConnectionBucketData): Promise<void> {
+    const updateFilter = this.updateDocFilter(data.user_id, data.client_id!);
+    await this.db.connection_report_events.findOneAndUpdate(
+      updateFilter,
+      {
+        $set: data,
+        $unset: {
+          disconnected_at: ''
+        }
+      },
+      {
+        upsert: true
+      }
+    );
+  }
+  async reportClientDisconnection(data: event_types.ClientDisconnectionEventData): Promise<void> {
+    const { connected_at, user_id, client_id } = data;
+    await this.db.connection_report_events.findOneAndUpdate(
+      {
+        client_id,
+        user_id,
+        connected_at
+      },
+      {
+        $set: {
+          disconnected_at: data.disconnected_at
+        },
+        $unset: {
+          jwt_exp: ''
+        }
+      }
+    );
+  }
+  async getConnectedClients(): Promise<event_types.ClientConnectionReportResponse> {
+    const result = await this.db.connection_report_events
+      .aggregate<event_types.ClientConnectionReportResponse>([
+        {
+          $match: {
+            disconnected_at: { $exists: false },
+            jwt_exp: { $gt: new Date() }
+          }
+        },
+        this.connectionsFacetPipeline(),
+        this.connectionsProjectPipeline()
+      ])
+      .toArray();
+    return result[0];
+  }
+
+  async [Symbol.asyncDispose]() {
+    // No-op
+  }
+
+  private parseJsDate(date: Date) {
+    const year = date.getUTCFullYear();
+    const month = date.getUTCMonth();
+    const today = date.getUTCDate();
+    const day = date.getUTCDay();
+    return {
+      year,
+      month,
+      today,
+      day,
+      parsedDate: date
+    };
+  }
+
+  private connectionsFacetPipeline() {
+    return {
+      $facet: {
+        unique_users: [
+          {
+            $group: {
+              _id: '$user_id'
+            }
+          },
+          {
+            $count: 'count'
+          }
+        ],
+        sdk_versions_array: [
+          {
+            $group: {
+              _id: '$sdk',
+              total: { $sum: 1 },
+              client_ids: { $addToSet: '$client_id' },
+              user_ids: { $addToSet: '$user_id' }
+            }
+          },
+          {
+            $project: {
+              _id: 0,
+              sdk: '$_id',
+              users: { $size: '$user_ids' },
+              clients: { $size: '$client_ids' }
+            }
+          },
+          {
+            $sort: {
+              sdk: 1
+            }
+          }
+        ]
+      }
+    };
+  }
+
+  private connectionsProjectPipeline() {
+    return {
+      $project: {
+        users: { $ifNull: [{ $arrayElemAt: ['$unique_users.count', 0] }, 0] },
+        sdks: '$sdk_versions_array'
+      }
+    };
+  }
+
+  private updateDocFilter(userId: string, clientId: string) {
+    const { year, month, today } = this.parseJsDate(new Date());
+    const nextDay = today + 1;
+    return {
+      user_id: userId,
+      client_id: clientId,
+      connected_at: {
+        $gte: new Date(Date.UTC(year, month, today)),
+        $lt: new Date(Date.UTC(year, month, nextDay))
+      }
+    };
+  }
+}

package/src/storage/implementation/MongoBucketBatch.ts

@@ -28,7 +28,7 @@ import { MongoIdSequence } from './MongoIdSequence.js';
 import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js';
 import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js';
 import { PersistedBatch } from './PersistedBatch.js';
-import { idPrefixFilter } from './util.js';
+import { idPrefixFilter } from '../../utils/util.js';
 
 /**
  * 15MB

package/src/storage/implementation/MongoStorageProvider.ts

@@ -4,8 +4,9 @@ import { POWERSYNC_VERSION, storage } from '@powersync/service-core';
 import { MongoStorageConfig } from '../../types/types.js';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
 import { PowerSyncMongo } from './db.js';
+import { MongoReportStorage } from '../MongoReportStorage.js';
 
-export class MongoStorageProvider implements storage.BucketStorageProvider {
+export class MongoStorageProvider implements storage.StorageProvider {
   get type() {
     return lib_mongo.MONGO_CONNECTION_TYPE;
   }
@@ -37,15 +38,19 @@ export class MongoStorageProvider implements storage.BucketStorageProvider {
     await client.connect();
 
     const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
-    const factory = new MongoBucketStorage(database, {
+    const syncStorageFactory = new MongoBucketStorage(database, {
       // TODO currently need the entire resolved config due to this
       slot_name_prefix: resolvedConfig.slot_name_prefix
     });
+
+    // Storage factory for reports
+    const reportStorageFactory = new MongoReportStorage(database);
     return {
-      storage: factory,
+      storage: syncStorageFactory,
+      reportStorage: reportStorageFactory,
       shutDown: async () => {
         shuttingDown = true;
-        await factory[Symbol.asyncDispose]();
+        await syncStorageFactory[Symbol.asyncDispose]();
         await client.close();
       },
       tearDown: () => {

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -37,7 +37,8 @@ import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js';
 import { MongoCompactor } from './MongoCompactor.js';
 import { MongoParameterCompactor } from './MongoParameterCompactor.js';
 import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
-import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from './util.js';
+import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../utils/util.js';
+
 
 export interface MongoSyncBucketStorageOptions {
   checksumOptions?: MongoChecksumOptions;

package/src/storage/implementation/PersistedBatch.ts

@@ -16,7 +16,7 @@ import {
   CurrentDataDocument,
   SourceKey
 } from './models.js';
-import { replicaIdToSubkey } from './util.js';
+import { replicaIdToSubkey } from '../../utils/util.js';
 
 /**
  * Maximum size of operations we write in a single transaction.

package/src/storage/implementation/db.ts

@@ -8,6 +8,7 @@ import {
   BucketParameterDocument,
   BucketStateDocument,
   CheckpointEventDocument,
+  ClientConnectionDocument,
   CurrentDataDocument,
   CustomWriteCheckpointDocument,
   IdSequenceDocument,
@@ -37,6 +38,7 @@ export class PowerSyncMongo {
   readonly locks: mongo.Collection<lib_mongo.locks.Lock>;
   readonly bucket_state: mongo.Collection<BucketStateDocument>;
   readonly checkpoint_events: mongo.Collection<CheckpointEventDocument>;
+  readonly connection_report_events: mongo.Collection<ClientConnectionDocument>;
 
   readonly client: mongo.MongoClient;
   readonly db: mongo.Db;
@@ -61,6 +63,7 @@ export class PowerSyncMongo {
     this.locks = this.db.collection('locks');
     this.bucket_state = this.db.collection('bucket_state');
     this.checkpoint_events = this.db.collection('checkpoint_events');
+    this.connection_report_events = this.db.collection('connection_report_events');
   }
 
   /**
@@ -128,6 +131,20 @@ export class PowerSyncMongo {
     });
   }
 
+  /**
+   * Only use in migrations and tests.
+   */
+  async createConnectionReportingCollection() {
+    const existingCollections = await this.db
+      .listCollections({ name: 'connection_report_events' }, { nameOnly: false })
+      .toArray();
+    const collection = existingCollections[0];
+    if (collection != null) {
+      return;
+    }
+    await this.db.createCollection('connection_report_events');
+  }
+
   /**
    * Only use in migrations and tests.
    */

package/src/storage/implementation/models.ts

@@ -1,6 +1,7 @@
 import { InternalOpId, storage } from '@powersync/service-core';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
+import { event_types } from '@powersync/service-types';
 
 /**
  * Replica id uniquely identifying a row on the source database.
@@ -238,3 +239,5 @@ export interface InstanceDocument {
   // The instance UUID
   _id: string;
 }
+
+export interface ClientConnectionDocument extends event_types.ClientConnection {}

package/src/storage/storage-index.ts

@@ -7,8 +7,9 @@ export * from './implementation/MongoPersistedSyncRulesContent.js';
 export * from './implementation/MongoStorageProvider.js';
 export * from './implementation/MongoSyncBucketStorage.js';
 export * from './implementation/MongoSyncRulesLock.js';
-export * from './implementation/MongoTestStorageFactoryGenerator.js';
 export * from './implementation/OperationBatch.js';
 export * from './implementation/PersistedBatch.js';
-export * from './implementation/util.js';
+export * from '../utils/util.js';
 export * from './MongoBucketStorage.js';
+export * from './MongoReportStorage.js';
+export * as test_utils from '../utils/test-utils.js';

package/src/utils/test-utils.ts

@@ -0,0 +1,57 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { PowerSyncMongo } from '../storage/implementation/db.js';
+import { TestStorageOptions } from '@powersync/service-core';
+import { MongoReportStorage } from '../storage/MongoReportStorage.js';
+import { MongoBucketStorage } from '../storage/MongoBucketStorage.js';
+import { MongoSyncBucketStorageOptions } from '../storage/implementation/MongoSyncBucketStorage.js';
+
+export type MongoTestStorageOptions = {
+  url: string;
+  isCI: boolean;
+  internalOptions?: MongoSyncBucketStorageOptions;
+};
+
+export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) {
+  return async (options?: TestStorageOptions) => {
+    const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI);
+
+    // None of the tests insert data into this collection, so it was never created
+    if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) {
+      await db.db.createCollection('bucket_parameters');
+    }
+
+    // Full migrations are not currently run for tests, so we manually create this
+    await db.createCheckpointEventsCollection();
+
+    if (!options?.doNotClear) {
+      await db.clear();
+    }
+
+    return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }, factoryOptions.internalOptions);
+  };
+}
+
+export function mongoTestReportStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) {
+  return async (options?: TestStorageOptions) => {
+    const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI);
+
+    await db.createConnectionReportingCollection();
+
+    if (!options?.doNotClear) {
+      await db.clear();
+    }
+
+    return new MongoReportStorage(db);
+  };
+}
+
+export const connectMongoForTests = (url: string, isCI: boolean) => {
+  // Short timeout for tests, to fail fast when the server is not available.
+  // Slightly longer timeouts for CI, to avoid arbitrary test failures
+  const client = new mongo.MongoClient(url, {
+    connectTimeoutMS: isCI ? 15_000 : 5_000,
+    socketTimeoutMS: isCI ? 15_000 : 5_000,
+    serverSelectionTimeoutMS: isCI ? 15_000 : 2_500
+  });
+  return new PowerSyncMongo(client);
+};

package/src/{storage/implementation → utils}/util.ts

@@ -3,11 +3,9 @@ import * as crypto from 'crypto';
 import * as uuid from 'uuid';
 
 import { mongo } from '@powersync/lib-service-mongodb';
-import { BucketChecksum, PartialChecksum, PartialOrFullChecksum, storage, utils } from '@powersync/service-core';
-
-import { PowerSyncMongo } from './db.js';
-import { BucketDataDocument } from './models.js';
+import { storage, utils } from '@powersync/service-core';
 import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { BucketDataDocument } from '../storage/implementation/models.js';
 
 export function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T> {
   let filter = {
@@ -105,20 +103,6 @@ export function replicaIdToSubkey(table: bson.ObjectId, id: storage.ReplicaId):
   }
 }
 
-/**
- * Helper for unit tests
- */
-export const connectMongoForTests = (url: string, isCI: boolean) => {
-  // Short timeout for tests, to fail fast when the server is not available.
-  // Slightly longer timeouts for CI, to avoid arbitrary test failures
-  const client = new mongo.MongoClient(url, {
-    connectTimeoutMS: isCI ? 15_000 : 5_000,
-    socketTimeoutMS: isCI ? 15_000 : 5_000,
-    serverSelectionTimeoutMS: isCI ? 15_000 : 2_500
-  });
-  return new PowerSyncMongo(client);
-};
-
 export function setSessionSnapshotTime(session: mongo.ClientSession, time: bson.Timestamp) {
   // This is a workaround for the lack of direct support for snapshot reads in the MongoDB driver.
   if (!session.snapshotEnabled) {

package/src/utils/utils-index.ts

@@ -0,0 +1,2 @@
+export * as test_utils from './test-utils.js';
+export * from './util.js';

package/test/src/__snapshots__/connection-report-storage.test.ts.snap

@@ -0,0 +1,215 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Connection reporting storage > Should create a connection report if its after a day 1`] = `
+[
+  {
+    "client_id": "client_week",
+    "sdk": "powersync-js/1.24.5",
+    "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+    "user_id": "user_week",
+  },
+  {
+    "client_id": "client_week",
+    "sdk": "powersync-js/1.24.5",
+    "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+    "user_id": "user_week",
+  },
+]
+`;
+
+exports[`Connection reporting storage > Should delete rows older than specified range 1`] = `
+{
+  "sdks": [
+    {
+      "clients": 1,
+      "sdk": "powersync-dart/1.6.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.1",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.2",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.24.5",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "unknown",
+      "users": 1,
+    },
+  ],
+  "users": 5,
+}
+`;
+
+exports[`Connection reporting storage > Should update a connected connection report and make it disconnected 1`] = `
+[
+  {
+    "client_id": "client_three",
+    "sdk": "powersync-js/1.21.2",
+    "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+    "user_id": "user_three",
+  },
+]
+`;
+
+exports[`Connection reporting storage > Should update a connection report if its within a day 1`] = `
+[
+  {
+    "client_id": "client_one",
+    "sdk": "powersync-dart/1.6.4",
+    "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android",
+    "user_id": "user_one",
+  },
+]
+`;
+
+exports[`Report storage tests > Should show connection report data for user over the past day 1`] = `
+{
+  "sdks": [
+    {
+      "clients": 1,
+      "sdk": "powersync-dart/1.6.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.1",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "unknown",
+      "users": 1,
+    },
+  ],
+  "users": 3,
+}
+`;
+
+exports[`Report storage tests > Should show connection report data for user over the past month 1`] = `
+{
+  "sdks": [
+    {
+      "clients": 1,
+      "sdk": "powersync-dart/1.6.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.1",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.2",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.23.6",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.23.7",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.24.5",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "unknown",
+      "users": 1,
+    },
+  ],
+  "users": 7,
+}
+`;
+
+exports[`Report storage tests > Should show connection report data for user over the past week 1`] = `
+{
+  "sdks": [
+    {
+      "clients": 1,
+      "sdk": "powersync-dart/1.6.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.1",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.2",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.24.5",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "unknown",
+      "users": 1,
+    },
+  ],
+  "users": 5,
+}
+`;
+
+exports[`Report storage tests > Should show currently connected users 1`] = `
+{
+  "sdks": [
+    {
+      "clients": 1,
+      "sdk": "powersync-dart/1.6.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.1",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "unknown",
+      "users": 1,
+    },
+  ],
+  "users": 2,
+}
+`;