@powersync/service-module-postgres-storage 0.0.0-dev-20260313100403 → 0.0.0-dev-20260515144844

package/src/storage/PostgresSyncRulesStorage.ts

@@ -9,6 +9,7 @@ import {
   internalToExternalOpId,
   LastValueSink,
   maxLsn,
+  ParameterSetLimitExceededError,
   PartialChecksum,
   PopulateChecksumCacheOptions,
   PopulateChecksumCacheResults,
@@ -22,7 +23,7 @@ import { JSONBig } from '@powersync/service-jsonbig';
 import * as sync_rules from '@powersync/service-sync-rules';
 import * as timers from 'timers/promises';
 import * as uuid from 'uuid';
-import { BIGINT_MAX } from '../types/codecs.js';
+import { bigint, BIGINT_MAX } from '../types/codecs.js';
 import { models, RequiredOperationBatchLimits } from '../types/types.js';
 import { replicaIdToSubkey } from '../utils/bson.js';
 import { mapOpEntry } from '../utils/bucket-data.js';
@@ -30,13 +31,14 @@ import { mapOpEntry } from '../utils/bucket-data.js';
 import * as framework from '@powersync/lib-services-framework';
 import { StatementParam } from '@powersync/service-jpgwire';
 import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
+import * as t from 'ts-codec';
 import { SourceTableDecoded, StoredRelationId } from '../types/models/SourceTable.js';
 import { pick } from '../utils/ts-codec.js';
 import { PostgresBucketBatch } from './batch/PostgresBucketBatch.js';
 import { PostgresWriteCheckpointAPI } from './checkpoints/PostgresWriteCheckpointAPI.js';
+import { PostgresCurrentDataStore } from './current-data-store.js';
 import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js';
 import { PostgresCompactor } from './PostgresCompactor.js';
-import { PostgresCurrentDataStore } from './current-data-store.js';
 
 export type PostgresSyncRulesStorageOptions = {
   factory: PostgresBucketStorageFactory;
@@ -50,11 +52,14 @@ export class PostgresSyncRulesStorage
   extends framework.BaseObserver<storage.SyncRulesBucketStorageListener>
   implements storage.SyncRulesBucketStorage
 {
+  [framework.DO_NOT_LOG] = true;
+
   public readonly group_id: number;
   public readonly sync_rules: storage.PersistedSyncRulesContent;
   public readonly slot_name: string;
   public readonly factory: PostgresBucketStorageFactory;
   public readonly storageConfig: StorageVersionConfig;
+  public readonly logger: framework.Logger;
 
   private sharedIterator = new BroadcastIterable((signal) => this.watchActiveCheckpoint(signal));
 
@@ -77,6 +82,7 @@ export class PostgresSyncRulesStorage
     this.factory = options.factory;
     this.storageConfig = options.sync_rules.getStorageConfig();
     this.currentDataStore = new PostgresCurrentDataStore(this.storageConfig);
+    this.logger = options.sync_rules.logger;
 
     this.writeCheckpointAPI = new PostgresWriteCheckpointAPI({
       db: this.db,
@@ -106,8 +112,8 @@ export class PostgresSyncRulesStorage
   getParsedSyncRules(options: storage.ParseSyncRulesOptions): sync_rules.HydratedSyncRules {
     const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {};
     /**
-     * Check if the cached sync rules, if present, had the same options.
-     * Parse sync rules if the options are different or if there is no cached value.
+     * Check if the cached sync config, if present, had the same options.
+     * Parse sync config if the options are different or if there is no cached value.
      */
     if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) {
       this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).hydratedSyncRules(), options };
@@ -137,11 +143,12 @@ export class PostgresSyncRulesStorage
 
     return new PostgresCompactor(this.db, this.group_id, {
       ...options,
-      maxOpId
+      maxOpId,
+      logger: this.logger
     }).compact();
   }
 
-  async populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults> {
+  async populatePersistentChecksumCache(_options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults> {
     // no-op - checksum cache is not implemented for Postgres yet
     return { buckets: 0 };
   }
@@ -361,7 +368,7 @@ export class PostgresSyncRulesStorage
     const checkpoint_lsn = syncRules?.last_checkpoint_lsn ?? null;
 
     const writer = new PostgresBucketBatch({
-      logger: options.logger ?? framework.logger,
+      logger: options.logger ?? this.logger,
       db: this.db,
       sync_rules: this.sync_rules.parsed(options).hydratedSyncRules(),
       group_id: this.group_id,
@@ -394,42 +401,69 @@ export class PostgresSyncRulesStorage
 
   async getParameterSets(
     checkpoint: ReplicationCheckpoint,
-    lookups: sync_rules.ScopedParameterLookup[]
-  ): Promise<sync_rules.SqliteJsonRow[]> {
+    lookups: sync_rules.ScopedParameterLookup[],
+    limit: number
+  ): Promise<sync_rules.ParameterLookupRows[]> {
     const rows = await this.db.sql`
-      SELECT DISTINCT
-        ON (lookup, source_table, source_key) lookup,
-        source_table,
-        source_key,
-        id,
-        bucket_parameters
-      FROM
-        bucket_parameters
-      WHERE
-        group_id = ${{ type: 'int4', value: this.group_id }}
-        AND lookup = ANY (
-          SELECT
-            decode((FILTER ->> 0)::text, 'hex') -- Decode the hex string to bytea
+      WITH
+        rows AS (
+          SELECT DISTINCT
+            ON (lookup, source_table, source_key) requested.index - 1 AS index,
+            bucket_parameters
           FROM
+            bucket_parameters,
             jsonb_array_elements(${{
         type: 'jsonb',
         value: lookups.map((l) => storage.serializeLookupBuffer(l).toString('hex'))
-      }}) AS FILTER
+      }}) WITH ORDINALITY AS requested (value, index)
+          WHERE
+            group_id = ${{ type: 'int4', value: this.group_id }}
+            AND lookup = decode((requested.value ->> 0)::text, 'hex') -- Decode the hex string to bytea
+            AND id <= ${{ type: 'int8', value: checkpoint.checkpoint }}
+          ORDER BY
+            lookup,
+            source_table,
+            source_key,
+            id DESC
         )
-        AND id <= ${{ type: 'int8', value: checkpoint.checkpoint }}
-      ORDER BY
-        lookup,
-        source_table,
-        source_key,
-        id DESC
+      SELECT
+        index,
+        bucket_parameters
+      FROM
+        rows
+      WHERE
+        bucket_parameters != '[]'
+      LIMIT
+        ${{ type: 'int4', value: limit + 1 }}
     `
-      .decoded(pick(models.BucketParameters, ['bucket_parameters']))
+      .decoded(parameterSetsRow)
       .rows();
 
-    const groupedParameters = rows.map((row) => {
-      return JSONBig.parse(row.bucket_parameters) as sync_rules.SqliteJsonRow;
-    });
-    return groupedParameters.flat();
+    let totalRows = 0;
+    const resultsByLookup = new Map<sync_rules.ScopedParameterLookup, sync_rules.SqliteJsonRow[]>();
+    for (const row of rows) {
+      const parameterRows = JSONBig.parse(row.bucket_parameters) as sync_rules.SqliteJsonRow[];
+      const lookup = lookups[Number(row.index)];
+      totalRows += parameterRows.length;
+
+      const existingResults = resultsByLookup.get(lookup);
+      if (existingResults != null) {
+        existingResults.push(...parameterRows);
+      } else {
+        resultsByLookup.set(lookup, parameterRows);
+      }
+    }
+
+    if (totalRows > limit) {
+      // Note that the LIMIT in the query allows more rows than parameters (because each row stores an array of
+      // parameter results). That array is very small though, and it doesn't allow fewer rows (due to the != []), so
+      // the SQL limit is good enough.
+      throw new ParameterSetLimitExceededError(limit);
+    }
+
+    const results: sync_rules.ParameterLookupRows[] = [];
+    resultsByLookup.forEach((rows, lookup) => results.push({ lookup, rows }));
+    return results;
   }
 
   async *getBucketDataBatch(
@@ -646,7 +680,7 @@ export class PostgresSyncRulesStorage
       .first();
 
     if (syncRulesRow == null) {
-      throw new Error('Cannot find sync rules status');
+      throw new Error('Cannot find replication stream status');
     }
 
     return {
@@ -839,7 +873,7 @@ export class PostgresSyncRulesStorage
 
     if (doc == null) {
       // Abort the connections - clients will have to retry later.
-      throw new framework.ServiceError(framework.ErrorCode.PSYNC_S2302, 'No active sync rules available');
+      throw new framework.ServiceError(framework.ErrorCode.PSYNC_S2302, 'No active replication stream available');
     }
 
     const sink = new LastValueSink<string>(undefined);
@@ -866,7 +900,7 @@ export class PostgresSyncRulesStorage
         continue;
       }
       if (Number(notification.active_checkpoint.id) != doc.id) {
-        // Active sync rules changed - abort and restart the stream
+        // Active replication stream changed - abort and restart the stream
         break;
       }
 
@@ -896,7 +930,15 @@ class PostgresReplicationCheckpoint implements storage.ReplicationCheckpoint {
     public readonly lsn: string | null
   ) {}
 
-  getParameterSets(lookups: sync_rules.ScopedParameterLookup[]): Promise<sync_rules.SqliteJsonRow[]> {
-    return this.storage.getParameterSets(this, lookups);
+  getParameterSets(
+    lookups: sync_rules.ScopedParameterLookup[],
+    limit: number
+  ): Promise<sync_rules.ParameterLookupRows[]> {
+    return this.storage.getParameterSets(this, lookups, limit);
   }
 }
+
+const parameterSetsRow = t.object({
+  index: bigint,
+  bucket_parameters: t.string
+});

package/src/storage/batch/PostgresBucketBatch.ts

@@ -20,16 +20,16 @@ import {
 import * as sync_rules from '@powersync/service-sync-rules';
 import * as timers from 'timers/promises';
 import * as t from 'ts-codec';
+import { bigint } from '../../types/codecs.js';
 import { CurrentBucket, V3CurrentDataDecoded } from '../../types/models/CurrentData.js';
 import { models, RequiredOperationBatchLimits } from '../../types/types.js';
 import { NOTIFICATION_CHANNEL } from '../../utils/db.js';
 import { pick } from '../../utils/ts-codec.js';
 import { batchCreateCustomWriteCheckpoints } from '../checkpoints/PostgresWriteCheckpointAPI.js';
-import { cacheKey, encodedCacheKey, OperationBatch, RecordOperation } from './OperationBatch.js';
-import { PostgresPersistedBatch } from './PostgresPersistedBatch.js';
-import { bigint } from '../../types/codecs.js';
 import { PostgresCurrentDataStore } from '../current-data-store.js';
 import { postgresTableId } from '../table-id.js';
+import { cacheKey, encodedCacheKey, OperationBatch, RecordOperation } from './OperationBatch.js';
+import { PostgresPersistedBatch } from './PostgresPersistedBatch.js';
 
 export interface PostgresBucketBatchOptions {
   logger: Logger;
@@ -52,7 +52,7 @@ export interface PostgresBucketBatchOptions {
 }
 
 /**
- * Intermediate type which helps for only watching the active sync rules
+ * Intermediate type which helps for only watching the active replication stream
  * via the Postgres NOTIFY protocol.
  */
 const StatefulCheckpoint = models.ActiveCheckpoint.and(t.object({ state: t.Enum(storage.SyncRuleState) }));
@@ -1054,7 +1054,7 @@ export class PostgresBucketBatch
       }
     });
     if (didActivate) {
-      this.logger.info(`Activated new sync rules at ${lsn}`);
+      this.logger.info(`Activated new replication stream at ${lsn}`);
     }
   }
 

package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts

@@ -64,7 +64,9 @@ export class PostgresWriteCheckpointAPI implements storage.WriteCheckpointAPI {
     switch (this.writeCheckpointMode) {
       case storage.WriteCheckpointMode.CUSTOM:
         if (false == 'sync_rules_id' in filters) {
-          throw new framework.errors.ValidationError(`Sync rules ID is required for custom Write Checkpoint filtering`);
+          throw new framework.errors.ValidationError(
+            `Replication stream ID is required for custom Write Checkpoint filtering`
+          );
         }
         return this.lastCustomWriteCheckpoint(filters as storage.CustomWriteCheckpointFilters);
       case storage.WriteCheckpointMode.MANAGED:

package/src/storage/current-data-store.ts

@@ -1,8 +1,8 @@
 import * as lib_postgres from '@powersync/lib-service-postgres';
 import { storage } from '@powersync/service-core';
 import * as t from 'ts-codec';
-import { pick } from '../utils/ts-codec.js';
 import * as models from '../types/models/CurrentData.js';
+import { pick } from '../utils/ts-codec.js';
 
 type Queryable = Pick<lib_postgres.DatabaseClient, 'sql' | 'streamRows'>;
 

package/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts

@@ -1,5 +1,5 @@
 import * as lib_postgres from '@powersync/lib-service-postgres';
-import { ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
 import { models } from '../../types/types.js';
 
@@ -31,17 +31,14 @@ export class PostgresPersistedSyncRulesContent extends storage.PersistedSyncRule
     });
     const lockHandle = await manager.acquire();
     if (!lockHandle) {
-      throw new ServiceError(
-        ErrorCode.PSYNC_S1003,
-        `Sync rules: ${this.id} have been locked by another process for replication.`
-      );
+      throw new ServiceError(ErrorCode.PSYNC_S1003, `Replication stream is locked by another process, standing by.`);
     }
 
     const interval = setInterval(async () => {
       try {
         await lockHandle.refresh();
       } catch (e) {
-        logger.error('Failed to refresh lock', e);
+        this.logger.error('Failed to refresh lock', e);
         clearInterval(interval);
       }
     }, 30_130);

package/src/types/models/SyncRules.ts

@@ -1,4 +1,5 @@
 import { framework, storage } from '@powersync/service-core';
+import { ReplicationError } from '@powersync/service-types';
 import * as t from 'ts-codec';
 import { bigint, pgwire_number } from '../codecs.js';
 import { jsonContainerObject } from './json.js';
@@ -59,7 +60,8 @@ export const SyncRules = t.object({
           overrides: t.record(t.boolean),
           maxTimeValuePrecision: t.number.optional()
         }),
-        eventDescriptors: t.record(t.array(t.string))
+        eventDescriptors: t.record(t.array(t.string)),
+        errors: t.array(ReplicationError).optional()
       })
     )
   )

package/src/types/models/models-index.ts

@@ -5,7 +5,7 @@ export * from './BucketParameters.js';
 export * from './CurrentData.js';
 export * from './Instance.js';
 export * from './Migration.js';
+export * from './SdkReporting.js';
 export * from './SourceTable.js';
 export * from './SyncRules.js';
 export * from './WriteCheckpoint.js';
-export * from './SdkReporting.js';

package/src/utils/test-utils.ts

@@ -1,9 +1,9 @@
+import { createLogger, logger as defaultLogger, transports } from '@powersync/lib-services-framework';
 import { framework, PowerSyncMigrationManager, ServiceContext, TestStorageOptions } from '@powersync/service-core';
 import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js';
-import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js';
-import { PostgresReportStorage } from '../storage/PostgresReportStorage.js';
 import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js';
-import { logger as defaultLogger, createLogger, transports } from '@powersync/lib-services-framework';
+import { PostgresReportStorage } from '../storage/PostgresReportStorage.js';
+import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js';
 import { truncateTables } from './db.js';
 
 export type PostgresTestStorageOptions = {

package/src/utils/utils-index.ts

@@ -1,5 +1,5 @@
 export * from './bson.js';
 export * from './bucket-data.js';
 export * from './db.js';
-export * from './ts-codec.js';
 export * as test_utils from './test-utils.js';
+export * from './ts-codec.js';

package/test/src/__snapshots__/{client-connections-storage.test.ts.snap → connection-report-storage.test.ts.snap}

@@ -19,7 +19,7 @@ exports[`Connection report storage > Should create a connection event if its aft
 
 exports[`Connection report storage > Should delete rows older than specified range 1`] = `
 {
-  "sdk_breakdown": [
+  "sdks": [
     {
       "clients": 1,
       "sdk": "powersync-dart/1.6.4",
@@ -51,7 +51,7 @@ exports[`Connection report storage > Should delete rows older than specified ran
       "users": 1,
     },
   ],
-  "total_users": 5,
+  "users": 5,
 }
 `;
 
@@ -77,9 +77,9 @@ exports[`Connection report storage > Should update a connection event if its wit
 ]
 `;
 
-exports[`Report storage tests > getClientConnectionsSummary returns the past day summary 1`] = `
+exports[`Report storage tests > Should show connection report data for user over the past day 1`] = `
 {
-  "sdk_breakdown": [
+  "sdks": [
     {
       "clients": 1,
       "sdk": "powersync-dart/1.6.4",
@@ -101,13 +101,13 @@ exports[`Report storage tests > getClientConnectionsSummary returns the past day
       "users": 1,
     },
   ],
-  "total_users": 3,
+  "users": 3,
 }
 `;
 
-exports[`Report storage tests > getClientConnectionsSummary returns the past month summary 1`] = `
+exports[`Report storage tests > Should show connection report data for user over the past month 1`] = `
 {
-  "sdk_breakdown": [
+  "sdks": [
     {
       "clients": 1,
       "sdk": "powersync-dart/1.6.4",
@@ -149,13 +149,13 @@ exports[`Report storage tests > getClientConnectionsSummary returns the past mon
       "users": 1,
     },
   ],
-  "total_users": 7,
+  "users": 7,
 }
 `;
 
-exports[`Report storage tests > getClientConnectionsSummary returns the past week summary 1`] = `
+exports[`Report storage tests > Should show connection report data for user over the past week 1`] = `
 {
-  "sdk_breakdown": [
+  "sdks": [
     {
       "clients": 1,
       "sdk": "powersync-dart/1.6.4",
@@ -187,13 +187,13 @@ exports[`Report storage tests > getClientConnectionsSummary returns the past wee
       "users": 1,
     },
   ],
-  "total_users": 5,
+  "users": 5,
 }
 `;
 
-exports[`Report storage tests > getCurrentConnections returns current connections 1`] = `
+exports[`Report storage tests > Should show currently connected users 1`] = `
 {
-  "sdk_breakdown": [
+  "sdks": [
     {
       "clients": 1,
       "sdk": "powersync-dart/1.6.4",
@@ -210,11 +210,11 @@ exports[`Report storage tests > getCurrentConnections returns current connection
       "users": 1,
     },
   ],
-  "total_users": 2,
+  "users": 2,
 }
 `;
 
-exports[`Report storage tests > getClientSessions returns sessions for a client_id 1`] = `
+exports[`Report storage tests > Should show paginated response of all connections of specified client_id 1`] = `
 {
   "count": 1,
   "cursor": undefined,
@@ -231,7 +231,7 @@ exports[`Report storage tests > getClientSessions returns sessions for a client_
 }
 `;
 
-exports[`Report storage tests > getClientSessions returns paginated sessions with a limit 1`] = `
+exports[`Report storage tests > Should show paginated response of all connections with a limit 1`] = `
 {
   "count": 4,
   "cursor": "<removed-for-snapshot>",
@@ -269,7 +269,7 @@ exports[`Report storage tests > getClientSessions returns paginated sessions wit
 }
 `;
 
-exports[`Report storage tests > getClientSessions returns paginated sessions with a limit 2`] = `
+exports[`Report storage tests > Should show paginated response of all connections with a limit 2`] = `
 {
   "count": 4,
   "cursor": undefined,
@@ -307,7 +307,7 @@ exports[`Report storage tests > getClientSessions returns paginated sessions wit
 }
 `;
 
-exports[`Report storage tests > getClientSessions returns paginated sessions with a limit and date range 1`] = `
+exports[`Report storage tests > Should show paginated response of all connections with a limit with date range 1`] = `
 {
   "count": 4,
   "cursor": "<removed-for-snapshot>",
@@ -345,7 +345,7 @@ exports[`Report storage tests > getClientSessions returns paginated sessions wit
 }
 `;
 
-exports[`Report storage tests > getClientSessions returns paginated sessions with a limit and date range 2`] = `
+exports[`Report storage tests > Should show paginated response of all connections with a limit with date range 2`] = `
 {
   "count": 2,
   "cursor": undefined,
@@ -383,7 +383,7 @@ exports[`Report storage tests > getClientSessions returns paginated sessions wit
 }
 `;
 
-exports[`Report storage tests > getClientSessions returns sessions for a user_id 1`] = `
+exports[`Report storage tests > Should show paginated response of connections of specified user_id 1`] = `
 {
   "count": 2,
   "cursor": undefined,
@@ -407,7 +407,7 @@ exports[`Report storage tests > getClientSessions returns sessions for a user_id
 }
 `;
 
-exports[`Report storage tests > getClientSessions returns sessions for a date range 1`] = `
+exports[`Report storage tests > Should show paginated response of connections over a date range 1`] = `
 {
   "count": 6,
   "cursor": undefined,
@@ -459,7 +459,7 @@ exports[`Report storage tests > getClientSessions returns sessions for a date ra
 }
 `;
 
-exports[`Report storage tests > getClientSessions returns sessions for client_id and user_id within a date range 1`] = `
+exports[`Report storage tests > Should show paginated response of connections over a date range of specified client_id and user_id 1`] = `
 {
   "count": 1,
   "cursor": undefined,