@powersync/service-module-postgres-storage 0.10.12 → 0.10.14

package/src/storage/PostgresReportStorage.ts

@@ -9,6 +9,7 @@ import { toInteger } from 'ix/util/tointeger.js';
 import { logger } from '@powersync/lib-services-framework';
 import { getStorageApplicationName } from '../utils/application-name.js';
 import { STORAGE_SCHEMA_NAME } from '../utils/db.js';
+import { ClientConnectionResponse } from '@powersync/service-types/dist/reports.js';
 
 export type PostgresReportStorageOptions = {
   config: NormalizedPostgresStorageConfig;
@@ -29,10 +30,10 @@ export class PostgresReportStorage implements storage.ReportStorage {
   }
 
   private parseJsDate(date: Date) {
-    const year = date.getFullYear();
-    const month = date.getMonth();
-    const today = date.getDate();
-    const day = date.getDay();
+    const year = date.getUTCFullYear();
+    const month = date.getUTCMonth();
+    const today = date.getUTCDate();
+    const day = date.getUTCDay();
     return {
       year,
       month,
@@ -106,8 +107,70 @@ export class PostgresReportStorage implements storage.ReportStorage {
     const { year, month, today } = this.parseJsDate(new Date());
     const nextDay = today + 1;
     return {
-      gte: new Date(year, month, today).toISOString(),
-      lt: new Date(year, month, nextDay).toISOString()
+      gte: new Date(Date.UTC(year, month, today)).toISOString(),
+      lt: new Date(Date.UTC(year, month, nextDay)).toISOString()
+    };
+  }
+
+  private clientsConnectionPagination(params: event_types.ClientConnectionAnalyticsRequest): {
+    mainQuery: pg_wire.Statement;
+    countQuery: pg_wire.Statement;
+  } {
+    const { cursor, limit, client_id, user_id, date_range } = params;
+    const queryLimit = limit || 100;
+    const queryParams: pg_wire.StatementParam[] = [];
+    let countQuery = `SELECT COUNT(*) AS total FROM connection_report_events`;
+    let query = `SELECT id, user_id, client_id, user_agent, sdk, jwt_exp::text AS jwt_exp, disconnected_at, connected_at::text AS connected_at, disconnected_at::text AS disconnected_at  FROM connection_report_events`;
+    let intermediateQuery = '';
+    /** Create a user_id/ client_id filter is they exist  */
+    if (client_id != null || user_id) {
+      if (client_id && !user_id) {
+        intermediateQuery += ` WHERE client_id = $1`;
+        queryParams.push({ type: 'varchar', value: client_id });
+      } else if (!client_id && user_id != null) {
+        intermediateQuery += ` WHERE user_id = $1`;
+        queryParams.push({ type: 'varchar', value: user_id });
+      } else {
+        intermediateQuery += ' WHERE client_id = $1 AND user_id = $2';
+        queryParams.push({ type: 'varchar', value: client_id! });
+        queryParams.push({ type: 'varchar', value: user_id! });
+      }
+    }
+
+    /** Create a date range filter if it exists */
+    if (date_range) {
+      const { start, end } = date_range;
+      intermediateQuery +=
+        queryParams.length === 0
+          ? ` WHERE connected_at >= $1 AND connected_at <= $2`
+          : ` AND connected_at >= $${queryParams.length + 1} AND connected_at <= $${queryParams.length + 2}`;
+      queryParams.push({ type: 1184, value: start.toISOString() });
+      queryParams.push({ type: 1184, value: end.toISOString() });
+    }
+
+    countQuery += intermediateQuery;
+
+    /** Create a cursor filter if it exists. The cursor in postgres is the last item connection date, the id is an uuid so we cant use the same logic as in MongoReportStorage.ts */
+    if (cursor) {
+      intermediateQuery +=
+        queryParams.length === 0 ? ` WHERE connected_at < $1` : ` AND connected_at < $${queryParams.length + 1}`;
+      queryParams.push({ type: 1184, value: new Date(cursor).toISOString() });
+    }
+
+    /** Order in descending connected at range to match Mongo sort=-1*/
+    intermediateQuery += ` ORDER BY connected_at DESC`;
+    query += intermediateQuery;
+
+    return {
+      mainQuery: {
+        statement: query,
+        params: queryParams,
+        limit: queryLimit
+      },
+      countQuery: {
+        statement: countQuery,
+        params: queryParams
+      }
     };
   }
 
@@ -225,6 +288,34 @@ export class PostgresReportStorage implements storage.ReportStorage {
       .first();
     return this.mapListCurrentConnectionsResponse(result);
   }
+
+  async getGeneralClientConnectionAnalytics(
+    data: event_types.ClientConnectionAnalyticsRequest
+  ): Promise<event_types.PaginatedResponse<event_types.ClientConnection>> {
+    const limit = data.limit || 100;
+    const statement = this.clientsConnectionPagination(data);
+
+    const result = await this.db.queryRows<ClientConnectionResponse>(statement.mainQuery);
+    const items = result.map((item) => ({
+      ...item,
+      /** JS Date conversion to match document schema used for Mongo storage */
+      connected_at: new Date(item.connected_at),
+      disconnected_at: item.disconnected_at ? new Date(item.disconnected_at) : undefined,
+      jwt_exp: item.jwt_exp ? new Date(item.jwt_exp) : undefined
+    }));
+    const count = items.length;
+    /** The returned total has been defaulted to 0 due to the overhead using documentCount from the mogo driver this is just to keep consistency with Mongo implementation.
+     * cursor.count has been deprecated.
+     * */
+    return {
+      items,
+      /** Setting the cursor to the connected at date of the last item in the list */
+      cursor: count === limit ? items[items.length - 1].connected_at.toISOString() : undefined,
+      count,
+      more: !(count !== limit)
+    };
+  }
+
   async deleteOldConnectionData(data: event_types.DeleteOldConnectionData): Promise<void> {
     const { date } = data;
     const result = await this.db.sql`

package/src/storage/batch/PostgresBucketBatch.ts

@@ -77,6 +77,7 @@ export class PostgresBucketBatch
   private lastWaitingLogThrottled = 0;
   private markRecordUnavailable: BucketStorageMarkRecordUnavailable | undefined;
   private needsActivation = true;
+  private clearedError = false;
 
   constructor(protected options: PostgresBucketBatchOptions) {
     super();
@@ -100,6 +101,10 @@ export class PostgresBucketBatch
     return this.last_checkpoint_lsn;
   }
 
+  get noCheckpointBeforeLsn() {
+    return this.no_checkpoint_before_lsn;
+  }
+
   async [Symbol.asyncDispose]() {
     super.clearListeners();
   }
@@ -213,7 +218,10 @@ export class PostgresBucketBatch
             });
           }
         }
-        await persistedBatch.flush(db);
+        const { flushedAny } = await persistedBatch.flush(db);
+        if (flushedAny) {
+          await this.clearError(db);
+        }
       });
     }
     if (processedCount == 0) {
@@ -570,6 +578,7 @@ export class PostgresBucketBatch
 
     // If set, we need to start a new transaction with this batch.
     let resumeBatch: OperationBatch | null = null;
+    let didFlush = false;
 
     // Now batch according to the sizes
     // This is a single batch if storeCurrentData == false
@@ -654,7 +663,8 @@ export class PostgresBucketBatch
         }
 
         if (persistedBatch!.shouldFlushTransaction()) {
-          await persistedBatch!.flush(db);
+          const { flushedAny } = await persistedBatch!.flush(db);
+          didFlush ||= flushedAny;
           // The operations stored in this batch will be processed in the `resumeBatch`
           persistedBatch = null;
           // Return the remaining entries for the next resume transaction
@@ -667,10 +677,15 @@ export class PostgresBucketBatch
          * The operations were less than the max size if here. Flush now.
          * `persistedBatch` will be `null` if the operations should be flushed in a new transaction.
          */
-        await persistedBatch.flush(db);
+        const { flushedAny } = await persistedBatch.flush(db);
+        didFlush ||= flushedAny;
       }
     }
 
+    if (didFlush) {
+      await this.clearError(db);
+    }
+
     // Don't return empty batches
     if (resumeBatch?.batch.length) {
       return resumeBatch;
@@ -1006,6 +1021,24 @@ export class PostgresBucketBatch
     }
   }
 
+  protected async clearError(
+    db: lib_postgres.AbstractPostgresConnection | lib_postgres.DatabaseClient = this.db
+  ): Promise<void> {
+    // No need to clear an error more than once per batch, since an error would always result in restarting the batch.
+    if (this.clearedError) {
+      return;
+    }
+
+    await db.sql`
+      UPDATE sync_rules
+      SET
+        last_fatal_error = ${{ type: 'varchar', value: null }}
+      WHERE
+        id = ${{ type: 'int4', value: this.group_id }}
+    `.execute();
+    this.clearedError = true;
+  }
+
   private async getLastOpIdSequence(db: lib_postgres.AbstractPostgresConnection) {
     // When no op_id has been generated, last_value = 1 and nextval() will be 1.
     // To cater for this case, we check is_called, and default to 0 if no value has been generated.

package/src/storage/batch/PostgresPersistedBatch.ts

@@ -236,6 +236,13 @@ export class PostgresPersistedBatch {
   }
 
   async flush(db: lib_postgres.WrappedConnection) {
+    const stats = {
+      bucketDataCount: this.bucketDataInserts.length,
+      parameterDataCount: this.parameterDataInserts.length,
+      currentDataCount: this.currentDataInserts.size + this.currentDataDeletes.length
+    };
+    const flushedAny = stats.bucketDataCount > 0 || stats.parameterDataCount > 0 || stats.currentDataCount > 0;
+
     logger.info(
       `powersync_${this.group_id} Flushed ${this.bucketDataInserts.length} + ${this.parameterDataInserts.length} + ${
         this.currentDataInserts.size + this.currentDataDeletes.length
@@ -251,6 +258,11 @@ export class PostgresPersistedBatch {
     this.currentDataDeletes = [];
     this.currentDataInserts = new Map();
     this.currentSize = 0;
+
+    return {
+      ...stats,
+      flushedAny
+    };
   }
 
   protected async flushBucketData(db: lib_postgres.WrappedConnection) {

package/test/src/__snapshots__/connection-report-storage.test.ts.snap

@@ -3,13 +3,13 @@
 exports[`Connection report storage > Should create a connection event if its after a day 1`] = `
 [
   {
-    "client_id": "client_week",
+    "client_id": "client_one",
     "sdk": "powersync-js/1.24.5",
     "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
     "user_id": "user_week",
   },
   {
-    "client_id": "client_week",
+    "client_id": "client_one",
     "sdk": "powersync-js/1.24.5",
     "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
     "user_id": "user_week",
@@ -213,3 +213,265 @@ exports[`Report storage tests > Should show currently connected users 1`] = `
   "users": 2,
 }
 `;
+
+exports[`Report storage tests > Should show paginated response of all connections of specified client_id 1`] = `
+{
+  "count": 1,
+  "cursor": undefined,
+  "items": [
+    {
+      "client_id": "client_two",
+      "id": "2",
+      "sdk": "powersync-js/1.21.1",
+      "user_agent": "powersync-js/1.21.0 powersync-web Chromium/138 linux",
+      "user_id": "user_two",
+    },
+  ],
+  "more": false,
+}
+`;
+
+exports[`Report storage tests > Should show paginated response of all connections with a limit 1`] = `
+{
+  "count": 4,
+  "cursor": "<removed-for-snapshot>",
+  "items": [
+    {
+      "client_id": "client_one",
+      "id": "1",
+      "sdk": "powersync-dart/1.6.4",
+      "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android",
+      "user_id": "user_one",
+    },
+    {
+      "client_id": "client_four",
+      "id": "4",
+      "sdk": "powersync-js/1.21.4",
+      "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+      "user_id": "user_four",
+    },
+    {
+      "client_id": "client_two",
+      "id": "2",
+      "sdk": "powersync-js/1.21.1",
+      "user_agent": "powersync-js/1.21.0 powersync-web Chromium/138 linux",
+      "user_id": "user_two",
+    },
+    {
+      "client_id": "",
+      "id": "5",
+      "sdk": "unknown",
+      "user_agent": "Dart (flutter-web) Chrome/128 android",
+      "user_id": "user_one",
+    },
+  ],
+  "more": true,
+}
+`;
+
+exports[`Report storage tests > Should show paginated response of all connections with a limit 2`] = `
+{
+  "count": 4,
+  "cursor": undefined,
+  "items": [
+    {
+      "client_id": "client_three",
+      "id": "3",
+      "sdk": "powersync-js/1.21.2",
+      "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+      "user_id": "user_three",
+    },
+    {
+      "client_id": "client_one",
+      "id": "week",
+      "sdk": "powersync-js/1.24.5",
+      "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+      "user_id": "user_week",
+    },
+    {
+      "client_id": "client_month",
+      "id": "month",
+      "sdk": "powersync-js/1.23.6",
+      "user_agent": "powersync-js/1.23.0 powersync-web Firefox/141 linux",
+      "user_id": "user_month",
+    },
+    {
+      "client_id": "client_expired",
+      "id": "expired",
+      "sdk": "powersync-js/1.23.7",
+      "user_agent": "powersync-js/1.23.0 powersync-web Firefox/141 linux",
+      "user_id": "user_expired",
+    },
+  ],
+  "more": false,
+}
+`;
+
+exports[`Report storage tests > Should show paginated response of all connections with a limit with date range 1`] = `
+{
+  "count": 4,
+  "cursor": "<removed-for-snapshot>",
+  "items": [
+    {
+      "client_id": "client_two",
+      "id": "2",
+      "sdk": "powersync-js/1.21.1",
+      "user_agent": "powersync-js/1.21.0 powersync-web Chromium/138 linux",
+      "user_id": "user_two",
+    },
+    {
+      "client_id": "",
+      "id": "5",
+      "sdk": "unknown",
+      "user_agent": "Dart (flutter-web) Chrome/128 android",
+      "user_id": "user_one",
+    },
+    {
+      "client_id": "client_three",
+      "id": "3",
+      "sdk": "powersync-js/1.21.2",
+      "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+      "user_id": "user_three",
+    },
+    {
+      "client_id": "client_one",
+      "id": "week",
+      "sdk": "powersync-js/1.24.5",
+      "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+      "user_id": "user_week",
+    },
+  ],
+  "more": true,
+}
+`;
+
+exports[`Report storage tests > Should show paginated response of all connections with a limit with date range 2`] = `
+{
+  "count": 2,
+  "cursor": undefined,
+  "items": [
+    {
+      "client_id": "client_two",
+      "id": "2",
+      "sdk": "powersync-js/1.21.1",
+      "user_agent": "powersync-js/1.21.0 powersync-web Chromium/138 linux",
+      "user_id": "user_two",
+    },
+    {
+      "client_id": "",
+      "id": "5",
+      "sdk": "unknown",
+      "user_agent": "Dart (flutter-web) Chrome/128 android",
+      "user_id": "user_one",
+    },
+    {
+      "client_id": "client_three",
+      "id": "3",
+      "sdk": "powersync-js/1.21.2",
+      "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+      "user_id": "user_three",
+    },
+    {
+      "client_id": "client_one",
+      "id": "week",
+      "sdk": "powersync-js/1.24.5",
+      "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+      "user_id": "user_week",
+    },
+  ],
+  "more": false,
+}
+`;
+
+exports[`Report storage tests > Should show paginated response of connections of specified user_id 1`] = `
+{
+  "count": 2,
+  "cursor": undefined,
+  "items": [
+    {
+      "client_id": "client_one",
+      "id": "1",
+      "sdk": "powersync-dart/1.6.4",
+      "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android",
+      "user_id": "user_one",
+    },
+    {
+      "client_id": "",
+      "id": "5",
+      "sdk": "unknown",
+      "user_agent": "Dart (flutter-web) Chrome/128 android",
+      "user_id": "user_one",
+    },
+  ],
+  "more": false,
+}
+`;
+
+exports[`Report storage tests > Should show paginated response of connections over a date range 1`] = `
+{
+  "count": 6,
+  "cursor": undefined,
+  "items": [
+    {
+      "client_id": "client_one",
+      "id": "1",
+      "sdk": "powersync-dart/1.6.4",
+      "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android",
+      "user_id": "user_one",
+    },
+    {
+      "client_id": "client_four",
+      "id": "4",
+      "sdk": "powersync-js/1.21.4",
+      "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+      "user_id": "user_four",
+    },
+    {
+      "client_id": "client_two",
+      "id": "2",
+      "sdk": "powersync-js/1.21.1",
+      "user_agent": "powersync-js/1.21.0 powersync-web Chromium/138 linux",
+      "user_id": "user_two",
+    },
+    {
+      "client_id": "",
+      "id": "5",
+      "sdk": "unknown",
+      "user_agent": "Dart (flutter-web) Chrome/128 android",
+      "user_id": "user_one",
+    },
+    {
+      "client_id": "client_three",
+      "id": "3",
+      "sdk": "powersync-js/1.21.2",
+      "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+      "user_id": "user_three",
+    },
+    {
+      "client_id": "client_one",
+      "id": "week",
+      "sdk": "powersync-js/1.24.5",
+      "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux",
+      "user_id": "user_week",
+    },
+  ],
+  "more": false,
+}
+`;
+
+exports[`Report storage tests > Should show paginated response of connections over a date range of specified client_id and user_id 1`] = `
+{
+  "count": 1,
+  "cursor": undefined,
+  "items": [
+    {
+      "client_id": "client_one",
+      "id": "1",
+      "sdk": "powersync-dart/1.6.4",
+      "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android",
+      "user_id": "user_one",
+    },
+  ],
+  "more": false,
+}
+`;

package/test/src/connection-report-storage.test.ts

@@ -185,7 +185,6 @@ describe('Connection report storage', async () => {
     const sdk = await factory.db
       .sql`SELECT * FROM connection_report_events WHERE user_id = ${{ type: 'varchar', value: userData.user_three.user_id }}`.rows<event_types.ClientConnection>();
     expect(sdk).toHaveLength(1);
-    console.log(sdk[0]);
     expect(new Date((sdk[0].disconnected_at! as unknown as DateTimeValue).iso8601Representation).toISOString()).toEqual(
       disconnectAt.toISOString()
     );