@powersync/service-module-postgres-storage 0.10.8 → 0.10.9

package/package.json

@@ -2,7 +2,7 @@
   "name": "@powersync/service-module-postgres-storage",
   "repository": "https://github.com/powersync-ja/powersync-service",
   "types": "dist/@types/index.d.ts",
-  "version": "0.10.8",
+  "version": "0.10.9",
   "main": "dist/index.js",
   "license": "FSL-1.1-ALv2",
   "type": "module",
@@ -29,17 +29,17 @@
     "p-defer": "^4.0.1",
     "ts-codec": "^1.3.0",
     "uuid": "^11.1.0",
-    "@powersync/lib-service-postgres": "0.4.12",
-    "@powersync/lib-services-framework": "0.7.8",
-    "@powersync/service-core": "1.15.8",
-    "@powersync/service-jpgwire": "0.21.4",
-    "@powersync/service-jsonbig": "0.17.11",
-    "@powersync/service-sync-rules": "0.29.5",
-    "@powersync/service-types": "0.13.0"
+    "@powersync/lib-service-postgres": "0.4.13",
+    "@powersync/lib-services-framework": "0.7.9",
+    "@powersync/service-core": "1.16.0",
+    "@powersync/service-types": "0.13.1",
+    "@powersync/service-jpgwire": "0.21.5",
+    "@powersync/service-jsonbig": "0.17.12",
+    "@powersync/service-sync-rules": "0.29.6"
   },
   "devDependencies": {
     "typescript": "^5.7.3",
-    "@powersync/service-core-tests": "0.12.8"
+    "@powersync/service-core-tests": "0.12.9"
   },
   "scripts": {
     "build": "tsc -b",

package/src/migrations/scripts/1756282360128-connection-reporting.ts

@@ -0,0 +1,41 @@
+import { migrations } from '@powersync/service-core';
+import { openMigrationDB } from '../migration-utils.js';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  await using client = openMigrationDB(configuration.storage);
+  await client.transaction(async (db) => {
+    await db.sql`
+      CREATE TABLE IF NOT EXISTS connection_report_events (
+        id TEXT PRIMARY KEY,
+        user_agent TEXT NOT NULL,
+        client_id TEXT NOT NULL,
+        user_id TEXT NOT NULL,
+        sdk TEXT NOT NULL,
+        jwt_exp TIMESTAMP WITH TIME ZONE,
+        connected_at TIMESTAMP WITH TIME ZONE NOT NULL,
+        disconnected_at TIMESTAMP WITH TIME ZONE
+      )
+    `.execute();
+
+    await db.sql`
+      CREATE INDEX IF NOT EXISTS sdk_list_index ON connection_report_events (connected_at, jwt_exp, disconnected_at)
+    `.execute();
+
+    await db.sql`CREATE INDEX IF NOT EXISTS sdk_user_id_index ON connection_report_events (user_id)`.execute();
+
+    await db.sql`CREATE INDEX IF NOT EXISTS sdk_client_id_index ON connection_report_events (client_id)`.execute();
+
+    await db.sql`CREATE INDEX IF NOT EXISTS sdk_index ON connection_report_events (sdk)`.execute();
+  });
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  await using client = openMigrationDB(configuration.storage);
+  await client.sql`DROP TABLE IF EXISTS connection_report_events`.execute();
+};

package/src/storage/PostgresReportStorage.ts

@@ -0,0 +1,258 @@
+import { storage } from '@powersync/service-core';
+import * as pg_wire from '@powersync/service-jpgwire';
+import { event_types } from '@powersync/service-types';
+import { v4 } from 'uuid';
+import * as lib_postgres from '@powersync/lib-service-postgres';
+import { NormalizedPostgresStorageConfig } from '../types/types.js';
+import { SdkReporting, SdkReportingDecoded } from '../types/models/SdkReporting.js';
+import { toInteger } from 'ix/util/tointeger.js';
+import { logger } from '@powersync/lib-services-framework';
+import { getStorageApplicationName } from '../utils/application-name.js';
+import { STORAGE_SCHEMA_NAME } from '../utils/db.js';
+
+export type PostgresReportStorageOptions = {
+  config: NormalizedPostgresStorageConfig;
+};
+
+export class PostgresReportStorage implements storage.ReportStorage {
+  readonly db: lib_postgres.DatabaseClient;
+  constructor(protected options: PostgresReportStorageOptions) {
+    this.db = new lib_postgres.DatabaseClient({
+      config: options.config,
+      schema: STORAGE_SCHEMA_NAME,
+      applicationName: getStorageApplicationName()
+    });
+
+    this.db.registerListener({
+      connectionCreated: async (connection) => this.prepareStatements(connection)
+    });
+  }
+
+  private parseJsDate(date: Date) {
+    const year = date.getFullYear();
+    const month = date.getMonth();
+    const today = date.getDate();
+    const day = date.getDay();
+    return {
+      year,
+      month,
+      today,
+      day,
+      parsedDate: date
+    };
+  }
+
+  private mapListCurrentConnectionsResponse(
+    result: SdkReportingDecoded | null
+  ): event_types.ClientConnectionReportResponse {
+    if (!result) {
+      return {
+        users: 0,
+        sdks: []
+      };
+    }
+    return {
+      users: Number(result.users),
+      sdks: result.sdks?.data || []
+    };
+  }
+  private async listConnectionsQuery() {
+    return await this.db.sql`
+      WITH
+        filtered AS (
+          SELECT
+            *
+          FROM
+            connection_report_events
+          WHERE
+            disconnected_at IS NULL
+            AND jwt_exp > NOW()
+        ),
+        unique_users AS (
+          SELECT
+            COUNT(DISTINCT user_id) AS count
+          FROM
+            filtered
+        ),
+        sdk_versions_array AS (
+          SELECT
+            sdk,
+            COUNT(DISTINCT client_id) AS clients,
+            COUNT(DISTINCT user_id) AS users
+          FROM
+            filtered
+          GROUP BY
+            sdk
+        )
+      SELECT
+        (
+          SELECT
+            COALESCE(count, 0)
+          FROM
+            unique_users
+        ) AS users,
+        (
+          SELECT
+            JSON_AGG(ROW_TO_JSON(s))
+          FROM
+            sdk_versions_array s
+        ) AS sdks;
+    `
+      .decoded(SdkReporting)
+      .first();
+  }
+
+  private updateTableFilter() {
+    const { year, month, today } = this.parseJsDate(new Date());
+    const nextDay = today + 1;
+    return {
+      gte: new Date(year, month, today).toISOString(),
+      lt: new Date(year, month, nextDay).toISOString()
+    };
+  }
+
+  async reportClientConnection(data: event_types.ClientConnectionBucketData): Promise<void> {
+    const { sdk, connected_at, user_id, user_agent, jwt_exp, client_id } = data;
+    const connectIsoString = connected_at.toISOString();
+    const jwtExpIsoString = jwt_exp.toISOString();
+    const { gte, lt } = this.updateTableFilter();
+    const uuid = v4();
+    const result = await this.db.sql`
+      UPDATE connection_report_events
+      SET
+        connected_at = ${{ type: 1184, value: connectIsoString }},
+        sdk = ${{ type: 'varchar', value: sdk }},
+        user_agent = ${{ type: 'varchar', value: user_agent }},
+        jwt_exp = ${{ type: 1184, value: jwtExpIsoString }},
+        disconnected_at = NULL
+      WHERE
+        user_id = ${{ type: 'varchar', value: user_id }}
+        AND client_id = ${{ type: 'varchar', value: client_id }}
+        AND connected_at >= ${{ type: 1184, value: gte }}
+        AND connected_at < ${{ type: 1184, value: lt }};
+    `.execute();
+    if (result.results[1].status === 'UPDATE 0') {
+      await this.db.sql`
+        INSERT INTO
+          connection_report_events (
+            user_id,
+            client_id,
+            connected_at,
+            sdk,
+            user_agent,
+            jwt_exp,
+            id
+          )
+        VALUES
+          (
+            ${{ type: 'varchar', value: user_id }},
+            ${{ type: 'varchar', value: client_id }},
+            ${{ type: 1184, value: connectIsoString }},
+            ${{ type: 'varchar', value: sdk }},
+            ${{ type: 'varchar', value: user_agent }},
+            ${{ type: 1184, value: jwtExpIsoString }},
+            ${{ type: 'varchar', value: uuid }}
+          )
+      `.execute();
+    }
+  }
+  async reportClientDisconnection(data: event_types.ClientDisconnectionEventData): Promise<void> {
+    const { user_id, client_id, disconnected_at, connected_at } = data;
+    const disconnectIsoString = disconnected_at.toISOString();
+    const connectIsoString = connected_at.toISOString();
+    await this.db.sql`
+      UPDATE connection_report_events
+      SET
+        disconnected_at = ${{ type: 1184, value: disconnectIsoString }},
+        jwt_exp = NULL
+      WHERE
+        user_id = ${{ type: 'varchar', value: user_id }}
+        AND client_id = ${{ type: 'varchar', value: client_id }}
+        AND connected_at = ${{ type: 1184, value: connectIsoString }}
+    `.execute();
+  }
+  async getConnectedClients(): Promise<event_types.ClientConnectionReportResponse> {
+    const result = await this.listConnectionsQuery();
+    return this.mapListCurrentConnectionsResponse(result);
+  }
+
+  async getClientConnectionReports(
+    data: event_types.ClientConnectionReportRequest
+  ): Promise<event_types.ClientConnectionReportResponse> {
+    const { start, end } = data;
+    const result = await this.db.sql`
+      WITH
+        filtered AS (
+          SELECT
+            *
+          FROM
+            connection_report_events
+          WHERE
+            connected_at >= ${{ type: 1184, value: start.toISOString() }}
+            AND connected_at <= ${{ type: 1184, value: end.toISOString() }}
+        ),
+        unique_users AS (
+          SELECT
+            COUNT(DISTINCT user_id) AS count
+          FROM
+            filtered
+        ),
+        sdk_versions_array AS (
+          SELECT
+            sdk,
+            COUNT(DISTINCT client_id) AS clients,
+            COUNT(DISTINCT user_id) AS users
+          FROM
+            filtered
+          GROUP BY
+            sdk
+        )
+      SELECT
+        (
+          SELECT
+            COALESCE(count, 0)
+          FROM
+            unique_users
+        ) AS users,
+        (
+          SELECT
+            JSON_AGG(ROW_TO_JSON(s))
+          FROM
+            sdk_versions_array s
+        ) AS sdks;
+    `
+      .decoded(SdkReporting)
+      .first();
+    return this.mapListCurrentConnectionsResponse(result);
+  }
+  async deleteOldConnectionData(data: event_types.DeleteOldConnectionData): Promise<void> {
+    const { date } = data;
+    const result = await this.db.sql`
+      DELETE FROM connection_report_events
+      WHERE
+        connected_at < ${{ type: 1184, value: date.toISOString() }}
+        AND (
+          disconnected_at IS NOT NULL
+          OR (
+            jwt_exp < NOW()
+            AND disconnected_at IS NULL
+          )
+        );
+    `.execute();
+    const deletedRows = toInteger(result.results[1].status.split(' ')[1] || '0');
+    if (deletedRows > 0) {
+      logger.info(
+        `TTL from ${date.toISOString()}: ${deletedRows} PostgresSQL rows have been removed from connection_report_events.`
+      );
+    }
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.db[Symbol.asyncDispose]();
+  }
+
+  async prepareStatements(connection: pg_wire.PgConnection) {
+    // It should be possible to prepare statements for some common operations here.
+    // This has not been implemented yet.
+  }
+}

package/src/storage/PostgresStorageProvider.ts

@@ -5,8 +5,9 @@ import { storage } from '@powersync/service-core';
 import { isPostgresStorageConfig, normalizePostgresStorageConfig, PostgresStorageConfig } from '../types/types.js';
 import { dropTables } from '../utils/db.js';
 import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js';
+import { PostgresReportStorage } from './PostgresReportStorage.js';
 
-export class PostgresStorageProvider implements storage.BucketStorageProvider {
+export class PostgresStorageProvider implements storage.StorageProvider {
   get type() {
     return lib_postgres.POSTGRES_CONNECTION_TYPE;
   }
@@ -28,13 +29,23 @@ export class PostgresStorageProvider implements storage.BucketStorageProvider {
       config: normalizedConfig,
       slot_name_prefix: options.resolvedConfig.slot_name_prefix
     });
+
+    const reportStorageFactory = new PostgresReportStorage({
+      config: normalizedConfig
+    });
+
     return {
+      reportStorage: reportStorageFactory,
       storage: storageFactory,
-      shutDown: async () => storageFactory.db[Symbol.asyncDispose](),
+      shutDown: async () => {
+        await storageFactory.db[Symbol.asyncDispose]();
+        await reportStorageFactory.db[Symbol.asyncDispose]();
+      },
       tearDown: async () => {
         logger.info(`Tearing down Postgres storage: ${normalizedConfig.database}...`);
         await dropTables(storageFactory.db);
         await storageFactory.db[Symbol.asyncDispose]();
+        await reportStorageFactory.db[Symbol.asyncDispose]();
         return true;
       }
     } satisfies storage.ActiveStorage;

package/src/storage/storage-index.ts

@@ -2,4 +2,3 @@ export * from './PostgresBucketStorageFactory.js';
 export * from './PostgresCompactor.js';
 export * from './PostgresStorageProvider.js';
 export * from './PostgresSyncRulesStorage.js';
-export * from './PostgresTestStorageFactoryGenerator.js';

package/src/types/models/SdkReporting.ts

@@ -0,0 +1,23 @@
+import * as t from 'ts-codec';
+import { bigint, jsonb } from '../codecs.js';
+
+export const Sdks = t.object({
+  sdk: t.string,
+  clients: t.number,
+  users: t.number
+});
+
+export type Sdks = t.Encoded<typeof Sdks>;
+
+export const SdkReporting = t.object({
+  users: bigint,
+  sdks: t
+    .object({
+      data: jsonb<Sdks[]>(t.array(Sdks))
+    })
+    .optional()
+    .or(t.Null)
+});
+
+export type SdkReporting = t.Encoded<typeof SdkReporting>;
+export type SdkReportingDecoded = t.Decoded<typeof SdkReporting>;

package/src/types/models/models-index.ts

@@ -8,3 +8,4 @@ export * from './Migration.js';
 export * from './SourceTable.js';
 export * from './SyncRules.js';
 export * from './WriteCheckpoint.js';
+export * from './SdkReporting.js';

package/src/{storage/PostgresTestStorageFactoryGenerator.ts → utils/test-utils.ts}

@@ -1,7 +1,8 @@
 import { framework, PowerSyncMigrationManager, ServiceContext, TestStorageOptions } from '@powersync/service-core';
 import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js';
 import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js';
-import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js';
+import { PostgresReportStorage } from '../storage/PostgresReportStorage.js';
+import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js';
 
 export type PostgresTestStorageOptions = {
   url: string;
@@ -12,7 +13,7 @@ export type PostgresTestStorageOptions = {
   migrationAgent?: (config: PostgresStorageConfigDecoded) => PostgresMigrationAgent;
 };
 
-export const postgresTestSetup = (factoryOptions: PostgresTestStorageOptions) => {
+export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
   const BASE_CONFIG = {
     type: 'postgresql' as const,
     uri: factoryOptions.url,
@@ -48,6 +49,21 @@ export const postgresTestSetup = (factoryOptions: PostgresTestStorageOptions) =>
   };
 
   return {
+    reportFactory: async (options?: TestStorageOptions) => {
+      try {
+        if (!options?.doNotClear) {
+          await migrate(framework.migrations.Direction.Up);
+        }
+
+        return new PostgresReportStorage({
+          config: TEST_CONNECTION_OPTIONS
+        });
+      } catch (ex) {
+        // Vitest does not display these errors nicely when using the `await using` syntx
+        console.error(ex, ex.cause);
+        throw ex;
+      }
+    },
     factory: async (options?: TestStorageOptions) => {
       try {
         if (!options?.doNotClear) {
@@ -66,8 +82,8 @@ export const postgresTestSetup = (factoryOptions: PostgresTestStorageOptions) =>
     },
     migrate
   };
-};
+}
 
-export const PostgresTestStorageFactoryGenerator = (factoryOptions: PostgresTestStorageOptions) => {
+export function postgresTestStorageFactoryGenerator(factoryOptions: PostgresTestStorageOptions) {
   return postgresTestSetup(factoryOptions).factory;
-};
+}

package/src/utils/utils-index.ts

@@ -2,3 +2,4 @@ export * from './bson.js';
 export * from './bucket-data.js';
 export * from './db.js';
 export * from './ts-codec.js';
+export * as test_utils from './test-utils.js';

package/test/src/__snapshots__/connection-report-storage.test.ts.snap

@@ -0,0 +1,215 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Connection report storage > Should create a connection event if its after a day 1`] = `
+[
+  {
+    "client_id": "client_week",
+    "sdk": "powersync-js/1.24.5",
+    "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+    "user_id": "user_week",
+  },
+  {
+    "client_id": "client_week",
+    "sdk": "powersync-js/1.24.5",
+    "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+    "user_id": "user_week",
+  },
+]
+`;
+
+exports[`Connection report storage > Should delete rows older than specified range 1`] = `
+{
+  "sdks": [
+    {
+      "clients": 1,
+      "sdk": "powersync-dart/1.6.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.1",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.2",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.24.5",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "unknown",
+      "users": 1,
+    },
+  ],
+  "users": 5,
+}
+`;
+
+exports[`Connection report storage > Should update a connection event and make it disconnected 1`] = `
+[
+  {
+    "client_id": "client_three",
+    "sdk": "powersync-js/1.21.2",
+    "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+    "user_id": "user_three",
+  },
+]
+`;
+
+exports[`Connection report storage > Should update a connection event if its within a day 1`] = `
+[
+  {
+    "client_id": "client_one",
+    "sdk": "powersync-dart/1.6.4",
+    "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android",
+    "user_id": "user_one",
+  },
+]
+`;
+
+exports[`Report storage tests > Should show connection report data for user over the past day 1`] = `
+{
+  "sdks": [
+    {
+      "clients": 1,
+      "sdk": "powersync-dart/1.6.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.1",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "unknown",
+      "users": 1,
+    },
+  ],
+  "users": 3,
+}
+`;
+
+exports[`Report storage tests > Should show connection report data for user over the past month 1`] = `
+{
+  "sdks": [
+    {
+      "clients": 1,
+      "sdk": "powersync-dart/1.6.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.1",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.2",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.23.6",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.23.7",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.24.5",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "unknown",
+      "users": 1,
+    },
+  ],
+  "users": 7,
+}
+`;
+
+exports[`Report storage tests > Should show connection report data for user over the past week 1`] = `
+{
+  "sdks": [
+    {
+      "clients": 1,
+      "sdk": "powersync-dart/1.6.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.1",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.2",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.24.5",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "unknown",
+      "users": 1,
+    },
+  ],
+  "users": 5,
+}
+`;
+
+exports[`Report storage tests > Should show currently connected users 1`] = `
+{
+  "sdks": [
+    {
+      "clients": 1,
+      "sdk": "powersync-dart/1.6.4",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "powersync-js/1.21.1",
+      "users": 1,
+    },
+    {
+      "clients": 1,
+      "sdk": "unknown",
+      "users": 1,
+    },
+  ],
+  "users": 2,
+}
+`;