@powersync/service-module-mongodb-storage 0.0.0-dev-20250828134335 → 0.0.0-dev-20250901073220

package/dist/{storage/implementation → utils}/util.d.ts

@@ -1,8 +1,7 @@
 import * as bson from 'bson';
 import { mongo } from '@powersync/lib-service-mongodb';
-import { PartialOrFullChecksum, storage, utils } from '@powersync/service-core';
-import { PowerSyncMongo } from './db.js';
-import { BucketDataDocument } from './models.js';
+import { storage, utils } from '@powersync/service-core';
+import { BucketDataDocument } from '../storage/implementation/models.js';
 export declare function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T>;
 export declare function generateSlotName(prefix: string, sync_rules_id: number): string;
 /**
@@ -22,35 +21,4 @@ export declare function readSingleBatch<T>(cursor: mongo.AbstractCursor<T>): Pro
 }>;
 export declare function mapOpEntry(row: BucketDataDocument): utils.OplogEntry;
 export declare function replicaIdToSubkey(table: bson.ObjectId, id: storage.ReplicaId): string;
-/**
- * Helper for unit tests
- */
-export declare const connectMongoForTests: (url: string, isCI: boolean) => PowerSyncMongo;
 export declare function setSessionSnapshotTime(session: mongo.ClientSession, time: bson.Timestamp): void;
-export declare const CHECKSUM_QUERY_GROUP_STAGE: {
-    $group: {
-        _id: string;
-        checksum_total: {
-            $sum: {
-                $toLong: string;
-            };
-        };
-        count: {
-            $sum: number;
-        };
-        has_clear_op: {
-            $max: {
-                $cond: (number | {
-                    $eq: string[];
-                })[];
-            };
-        };
-        last_op: {
-            $max: string;
-        };
-    };
-};
-/**
- * Convert output of CHECKSUM_QUERY_GROUP_STAGE into a checksum.
- */
-export declare function checksumFromAggregate(doc: bson.Document): PartialOrFullChecksum;

package/dist/{storage/implementation → utils}/util.js

@@ -1,9 +1,7 @@
 import * as bson from 'bson';
 import * as crypto from 'crypto';
 import * as uuid from 'uuid';
-import { mongo } from '@powersync/lib-service-mongodb';
 import { storage, utils } from '@powersync/service-core';
-import { PowerSyncMongo } from './db.js';
 import { ServiceAssertionError } from '@powersync/lib-services-framework';
 export function idPrefixFilter(prefix, rest) {
     let filter = {
@@ -96,19 +94,6 @@ export function replicaIdToSubkey(table, id) {
         return uuid.v5(repr, utils.ID_NAMESPACE);
     }
 }
-/**
- * Helper for unit tests
- */
-export const connectMongoForTests = (url, isCI) => {
-    // Short timeout for tests, to fail fast when the server is not available.
-    // Slightly longer timeouts for CI, to avoid arbitrary test failures
-    const client = new mongo.MongoClient(url, {
-        connectTimeoutMS: isCI ? 15_000 : 5_000,
-        socketTimeoutMS: isCI ? 15_000 : 5_000,
-        serverSelectionTimeoutMS: isCI ? 15_000 : 2_500
-    });
-    return new PowerSyncMongo(client);
-};
 export function setSessionSnapshotTime(session, time) {
     // This is a workaround for the lack of direct support for snapshot reads in the MongoDB driver.
     if (!session.snapshotEnabled) {
@@ -121,43 +106,4 @@ export function setSessionSnapshotTime(session, time) {
         throw new ServiceAssertionError(`Session snapshotTime is already set`);
     }
 }
-export const CHECKSUM_QUERY_GROUP_STAGE = {
-    $group: {
-        _id: '$_id.b',
-        // Historically, checksum may be stored as 'int' or 'double'.
-        // More recently, this should be a 'long'.
-        // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations.
-        checksum_total: { $sum: { $toLong: '$checksum' } },
-        count: { $sum: 1 },
-        has_clear_op: {
-            $max: {
-                $cond: [{ $eq: ['$op', 'CLEAR'] }, 1, 0]
-            }
-        },
-        last_op: { $max: '$_id.o' }
-    }
-};
-/**
- * Convert output of CHECKSUM_QUERY_GROUP_STAGE into a checksum.
- */
-export function checksumFromAggregate(doc) {
-    const partialChecksum = Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff;
-    const bucket = doc._id;
-    if (doc.has_clear_op == 1) {
-        return {
-            // full checksum - replaces any previous one
-            bucket,
-            checksum: partialChecksum,
-            count: doc.count
-        };
-    }
-    else {
-        return {
-            // partial checksum - is added to a previous one
-            bucket,
-            partialCount: doc.count,
-            partialChecksum
-        };
-    }
-}
 //# sourceMappingURL=util.js.map

package/dist/utils/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/utils/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAG7B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAG1E,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA+B;IACtE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,EAAqB;IAC3E,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,OAA4B,EAAE,IAAoB;IACvF,gGAAgG;IAChG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,oCAAoC,CAAC,CAAC;IACxE,CAAC;IACD,IAAK,OAAe,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;QACzC,OAAe,CAAC,YAAY,GAAG,IAAI,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;IACzE,CAAC;AACH,CAAC"}

package/dist/utils/utils-index.d.ts

@@ -0,0 +1,2 @@
+export * as test_utils from './test-utils.js';
+export * from './util.js';

package/dist/utils/utils-index.js

@@ -0,0 +1,3 @@
+export * as test_utils from './test-utils.js';
+export * from './util.js';
+//# sourceMappingURL=utils-index.js.map

package/dist/utils/utils-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils-index.js","sourceRoot":"","sources":["../../src/utils/utils-index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,UAAU,MAAM,iBAAiB,CAAC;AAC9C,cAAc,WAAW,CAAC"}

package/package.json

@@ -2,7 +2,7 @@
   "name": "@powersync/service-module-mongodb-storage",
   "repository": "https://github.com/powersync-ja/powersync-service",
   "types": "dist/index.d.ts",
-  "version": "0.0.0-dev-20250828134335",
+  "version": "0.0.0-dev-20250901073220",
   "main": "dist/index.js",
   "license": "FSL-1.1-ALv2",
   "type": "module",
@@ -27,15 +27,15 @@
     "lru-cache": "^10.2.2",
     "ts-codec": "^1.3.0",
     "uuid": "^11.1.0",
-    "@powersync/lib-service-mongodb": "0.0.0-dev-20250828134335",
-    "@powersync/lib-services-framework": "0.7.3",
-    "@powersync/service-core": "0.0.0-dev-20250828134335",
+    "@powersync/lib-service-mongodb": "0.0.0-dev-20250901073220",
+    "@powersync/lib-services-framework": "0.0.0-dev-20250901073220",
+    "@powersync/service-core": "0.0.0-dev-20250901073220",
+    "@powersync/service-types": "0.0.0-dev-20250901073220",
     "@powersync/service-jsonbig": "0.17.11",
-    "@powersync/service-sync-rules": "0.29.0",
-    "@powersync/service-types": "0.13.0"
+    "@powersync/service-sync-rules": "0.29.0"
   },
   "devDependencies": {
-    "@powersync/service-core-tests": "0.0.0-dev-20250828134335"
+    "@powersync/service-core-tests": "0.0.0-dev-20250901073220"
   },
   "scripts": {
     "build": "tsc -b",

package/src/index.ts

@@ -5,3 +5,4 @@ export * as storage from './storage/storage-index.js';
 
 export * from './types/types.js';
 export * as types from './types/types.js';
+export * as utils from './utils/utils-index.js';

package/src/migrations/db/migrations/1752661449910-connection-reporting.ts

@@ -0,0 +1,58 @@
+import { migrations } from '@powersync/service-core';
+import * as storage from '../../../storage/storage-index.js';
+import { MongoStorageConfig } from '../../../types/types.js';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+
+  try {
+    await db.createConnectionReportingCollection();
+
+    await db.connection_report_events.createIndex(
+      {
+        connected_at: 1,
+        jwt_exp: 1,
+        disconnected_at: 1
+      },
+      { name: 'connection_list_index' }
+    );
+
+    await db.connection_report_events.createIndex(
+      {
+        user_id: 1
+      },
+      { name: 'connection_user_id_index' }
+    );
+    await db.connection_report_events.createIndex(
+      {
+        client_id: 1
+      },
+      { name: 'connection_client_id_index' }
+    );
+    await db.connection_report_events.createIndex(
+      {
+        sdk: 1
+      },
+      { name: 'connection_index' }
+    );
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+
+  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+
+  try {
+    await db.db.dropCollection('connection_report_events');
+  } finally {
+    await db.client.close();
+  }
+};

package/src/storage/MongoBucketStorage.ts

@@ -12,7 +12,7 @@ import { PowerSyncMongo } from './implementation/db.js';
 import { SyncRuleDocument } from './implementation/models.js';
 import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js';
 import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js';
-import { generateSlotName } from './implementation/util.js';
+import { generateSlotName } from '../utils/util.js';
 
 export class MongoBucketStorage
   extends BaseObserver<storage.BucketStorageFactoryListener>

package/src/storage/MongoReportStorage.ts

@@ -0,0 +1,177 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { storage } from '@powersync/service-core';
+import { event_types } from '@powersync/service-types';
+import { PowerSyncMongo } from './implementation/db.js';
+import { logger } from '@powersync/lib-services-framework';
+
+export class MongoReportStorage implements storage.ReportStorage {
+  private readonly client: mongo.MongoClient;
+  public readonly db: PowerSyncMongo;
+
+  constructor(db: PowerSyncMongo) {
+    this.client = db.client;
+    this.db = db;
+  }
+  async deleteOldConnectionData(data: event_types.DeleteOldConnectionData): Promise<void> {
+    const { date } = data;
+    const result = await this.db.connection_report_events.deleteMany({
+      connected_at: { $lt: date },
+      $or: [
+        { disconnected_at: { $exists: true } },
+        { jwt_exp: { $lt: new Date() }, disconnected_at: { $exists: false } }
+      ]
+    });
+    if (result.deletedCount > 0) {
+      logger.info(
+        `TTL from ${date.toISOString()}: ${result.deletedCount} MongoDB documents have been removed from connection_report_events.`
+      );
+    }
+  }
+
+  async getClientConnectionReports(
+    data: event_types.ClientConnectionReportRequest
+  ): Promise<event_types.ClientConnectionReportResponse> {
+    const { start, end } = data;
+    const result = await this.db.connection_report_events
+      .aggregate<event_types.ClientConnectionReportResponse>([
+        {
+          $match: {
+            connected_at: { $lte: end, $gte: start }
+          }
+        },
+        this.connectionsFacetPipeline(),
+        this.connectionsProjectPipeline()
+      ])
+      .toArray();
+    return result[0];
+  }
+
+  async reportClientConnection(data: event_types.ClientConnectionBucketData): Promise<void> {
+    const updateFilter = this.updateDocFilter(data.user_id, data.client_id!);
+    await this.db.connection_report_events.findOneAndUpdate(
+      updateFilter,
+      {
+        $set: data,
+        $unset: {
+          disconnected_at: ''
+        }
+      },
+      {
+        upsert: true
+      }
+    );
+  }
+  async reportClientDisconnection(data: event_types.ClientDisconnectionEventData): Promise<void> {
+    const { connected_at, user_id, client_id } = data;
+    await this.db.connection_report_events.findOneAndUpdate(
+      {
+        client_id,
+        user_id,
+        connected_at
+      },
+      {
+        $set: {
+          disconnected_at: data.disconnected_at
+        },
+        $unset: {
+          jwt_exp: ''
+        }
+      }
+    );
+  }
+  async getConnectedClients(): Promise<event_types.ClientConnectionReportResponse> {
+    const result = await this.db.connection_report_events
+      .aggregate<event_types.ClientConnectionReportResponse>([
+        {
+          $match: {
+            disconnected_at: { $exists: false },
+            jwt_exp: { $gt: new Date() }
+          }
+        },
+        this.connectionsFacetPipeline(),
+        this.connectionsProjectPipeline()
+      ])
+      .toArray();
+    return result[0];
+  }
+
+  async [Symbol.asyncDispose]() {
+    // No-op
+  }
+
+  private parseJsDate(date: Date) {
+    const year = date.getUTCFullYear();
+    const month = date.getUTCMonth();
+    const today = date.getUTCDate();
+    const day = date.getUTCDay();
+    return {
+      year,
+      month,
+      today,
+      day,
+      parsedDate: date
+    };
+  }
+
+  private connectionsFacetPipeline() {
+    return {
+      $facet: {
+        unique_users: [
+          {
+            $group: {
+              _id: '$user_id'
+            }
+          },
+          {
+            $count: 'count'
+          }
+        ],
+        sdk_versions_array: [
+          {
+            $group: {
+              _id: '$sdk',
+              total: { $sum: 1 },
+              client_ids: { $addToSet: '$client_id' },
+              user_ids: { $addToSet: '$user_id' }
+            }
+          },
+          {
+            $project: {
+              _id: 0,
+              sdk: '$_id',
+              users: { $size: '$user_ids' },
+              clients: { $size: '$client_ids' }
+            }
+          },
+          {
+            $sort: {
+              sdk: 1
+            }
+          }
+        ]
+      }
+    };
+  }
+
+  private connectionsProjectPipeline() {
+    return {
+      $project: {
+        users: { $ifNull: [{ $arrayElemAt: ['$unique_users.count', 0] }, 0] },
+        sdks: '$sdk_versions_array'
+      }
+    };
+  }
+
+  private updateDocFilter(userId: string, clientId: string) {
+    const { year, month, today } = this.parseJsDate(new Date());
+    const nextDay = today + 1;
+    return {
+      user_id: userId,
+      client_id: clientId,
+      connected_at: {
+        $gte: new Date(Date.UTC(year, month, today)),
+        $lt: new Date(Date.UTC(year, month, nextDay))
+      }
+    };
+  }
+}

package/src/storage/implementation/MongoBucketBatch.ts

@@ -28,7 +28,7 @@ import { MongoIdSequence } from './MongoIdSequence.js';
 import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js';
 import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js';
 import { PersistedBatch } from './PersistedBatch.js';
-import { idPrefixFilter } from './util.js';
+import { idPrefixFilter } from '../../utils/util.js';
 
 /**
  * 15MB