@powersync/service-module-postgres-storage 0.0.0-dev-20260225160713 → 0.0.0-dev-20260313100403

package/src/types/models/CurrentData.ts

@@ -1,5 +1,5 @@
 import * as t from 'ts-codec';
-import { hexBuffer, jsonb, pgwire_number } from '../codecs.js';
+import { bigint, hexBuffer, jsonb, pgwire_number } from '../codecs.js';
 
 export const CurrentBucket = t.object({
   bucket: t.string,
@@ -10,7 +10,7 @@ export const CurrentBucket = t.object({
 export type CurrentBucket = t.Encoded<typeof CurrentBucket>;
 export type CurrentBucketDecoded = t.Decoded<typeof CurrentBucket>;
 
-export const CurrentData = t.object({
+export const V1CurrentData = t.object({
   buckets: jsonb(t.array(CurrentBucket)),
   data: hexBuffer,
   group_id: pgwire_number,
@@ -19,5 +19,18 @@ export const CurrentData = t.object({
   source_table: t.string
 });
 
-export type CurrentData = t.Encoded<typeof CurrentData>;
-export type CurrentDataDecoded = t.Decoded<typeof CurrentData>;
+export const V3CurrentData = t.object({
+  buckets: jsonb(t.array(CurrentBucket)),
+  data: hexBuffer,
+  group_id: pgwire_number,
+  lookups: t.array(hexBuffer),
+  source_key: hexBuffer,
+  source_table: t.string,
+  pending_delete: t.Null.or(bigint)
+});
+
+export type V1CurrentData = t.Encoded<typeof V1CurrentData>;
+export type V1CurrentDataDecoded = t.Decoded<typeof V1CurrentData>;
+
+export type V3CurrentData = t.Encoded<typeof V3CurrentData>;
+export type V3CurrentDataDecoded = t.Decoded<typeof V3CurrentData>;

package/src/utils/bson.ts

@@ -6,7 +6,7 @@ import * as uuid from 'uuid';
  * JSONB columns do not directly support storing binary data which could be required in future.
  */
 
-export function replicaIdToSubkey(tableId: string, id: storage.ReplicaId): string {
+export function replicaIdToSubkey(tableId: storage.SourceTableId, id: storage.ReplicaId): string {
   // Hashed UUID from the table and id
   if (storage.isUUID(id)) {
     // Special case for UUID for backwards-compatiblity

package/src/utils/db.ts

@@ -21,6 +21,7 @@ export const dropTables = async (client: lib_postgres.DatabaseClient) => {
     await db.sql`DROP TABLE IF EXISTS instance`.execute();
     await db.sql`DROP TABLE IF EXISTS bucket_data`.execute();
     await db.sql`DROP TABLE IF EXISTS current_data`.execute();
+    await db.sql`DROP TABLE IF EXISTS v3_current_data`.execute();
     await db.sql`DROP TABLE IF EXISTS source_tables`.execute();
     await db.sql`DROP TABLE IF EXISTS write_checkpoints`.execute();
     await db.sql`DROP TABLE IF EXISTS custom_write_checkpoints`.execute();
@@ -50,6 +51,15 @@ export const truncateTables = async (db: lib_postgres.DatabaseClient) => {
         connection_report_events RESTART IDENTITY CASCADE
     `
     },
+    {
+      // TRUNCATE if v3_current_data exists
+      statement: `DO $$
+        BEGIN
+          IF to_regclass('v3_current_data') IS NOT NULL THEN
+            EXECUTE 'TRUNCATE TABLE v3_current_data RESTART IDENTITY CASCADE';
+          END IF;
+        END $$;`
+    },
     {
       statement: `ALTER SEQUENCE IF EXISTS op_id_sequence RESTART
       WITH

package/src/utils/test-utils.ts

@@ -3,6 +3,7 @@ import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js'
 import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js';
 import { PostgresReportStorage } from '../storage/PostgresReportStorage.js';
 import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js';
+import { logger as defaultLogger, createLogger, transports } from '@powersync/lib-services-framework';
 import { truncateTables } from './db.js';
 
 export type PostgresTestStorageOptions = {
@@ -32,12 +33,20 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
 
     const mockServiceContext = { configuration: { storage: BASE_CONFIG } } as unknown as ServiceContext;
 
+    // Migration logs can get really verbose in tests, so only log warnings and up.
+    const logger = createLogger({
+      level: 'warn',
+      format: defaultLogger.format,
+      transports: [new transports.Console()]
+    });
+
     if (options.down) {
       await migrationManager.migrate({
         direction: framework.migrations.Direction.Down,
         migrationContext: {
           service_context: mockServiceContext
-        }
+        },
+        logger
       });
     }
 
@@ -46,7 +55,8 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
         direction: framework.migrations.Direction.Up,
         migrationContext: {
           service_context: mockServiceContext
-        }
+        },
+        logger
       });
     }
   };
@@ -100,10 +110,7 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
         throw ex;
       }
     },
-    migrate
+    migrate,
+    tableIdStrings: true
   };
 }
-
-export function postgresTestStorageFactoryGenerator(factoryOptions: PostgresTestStorageOptions) {
-  return postgresTestSetup(factoryOptions).factory;
-}

package/test/src/__snapshots__/{connection-report-storage.test.ts.snap → client-connections-storage.test.ts.snap}

@@ -19,7 +19,7 @@ exports[`Connection report storage > Should create a connection event if its aft
 
 exports[`Connection report storage > Should delete rows older than specified range 1`] = `
 {
-  "sdks": [
+  "sdk_breakdown": [
     {
       "clients": 1,
       "sdk": "powersync-dart/1.6.4",
@@ -51,7 +51,7 @@ exports[`Connection report storage > Should delete rows older than specified ran
       "users": 1,
     },
   ],
-  "users": 5,
+  "total_users": 5,
 }
 `;
 
@@ -77,9 +77,9 @@ exports[`Connection report storage > Should update a connection event if its wit
 ]
 `;
 
-exports[`Report storage tests > Should show connection report data for user over the past day 1`] = `
+exports[`Report storage tests > getClientConnectionsSummary returns the past day summary 1`] = `
 {
-  "sdks": [
+  "sdk_breakdown": [
     {
       "clients": 1,
       "sdk": "powersync-dart/1.6.4",
@@ -101,13 +101,13 @@ exports[`Report storage tests > Should show connection report data for user over
       "users": 1,
     },
   ],
-  "users": 3,
+  "total_users": 3,
 }
 `;
 
-exports[`Report storage tests > Should show connection report data for user over the past month 1`] = `
+exports[`Report storage tests > getClientConnectionsSummary returns the past month summary 1`] = `
 {
-  "sdks": [
+  "sdk_breakdown": [
     {
       "clients": 1,
       "sdk": "powersync-dart/1.6.4",
@@ -149,13 +149,13 @@ exports[`Report storage tests > Should show connection report data for user over
       "users": 1,
     },
   ],
-  "users": 7,
+  "total_users": 7,
 }
 `;
 
-exports[`Report storage tests > Should show connection report data for user over the past week 1`] = `
+exports[`Report storage tests > getClientConnectionsSummary returns the past week summary 1`] = `
 {
-  "sdks": [
+  "sdk_breakdown": [
     {
       "clients": 1,
       "sdk": "powersync-dart/1.6.4",
@@ -187,13 +187,13 @@ exports[`Report storage tests > Should show connection report data for user over
       "users": 1,
     },
   ],
-  "users": 5,
+  "total_users": 5,
 }
 `;
 
-exports[`Report storage tests > Should show currently connected users 1`] = `
+exports[`Report storage tests > getCurrentConnections returns current connections 1`] = `
 {
-  "sdks": [
+  "sdk_breakdown": [
     {
       "clients": 1,
       "sdk": "powersync-dart/1.6.4",
@@ -210,11 +210,11 @@ exports[`Report storage tests > Should show currently connected users 1`] = `
       "users": 1,
     },
   ],
-  "users": 2,
+  "total_users": 2,
 }
 `;
 
-exports[`Report storage tests > Should show paginated response of all connections of specified client_id 1`] = `
+exports[`Report storage tests > getClientSessions returns sessions for a client_id 1`] = `
 {
   "count": 1,
   "cursor": undefined,
@@ -231,7 +231,7 @@ exports[`Report storage tests > Should show paginated response of all connection
 }
 `;
 
-exports[`Report storage tests > Should show paginated response of all connections with a limit 1`] = `
+exports[`Report storage tests > getClientSessions returns paginated sessions with a limit 1`] = `
 {
   "count": 4,
   "cursor": "<removed-for-snapshot>",
@@ -269,7 +269,7 @@ exports[`Report storage tests > Should show paginated response of all connection
 }
 `;
 
-exports[`Report storage tests > Should show paginated response of all connections with a limit 2`] = `
+exports[`Report storage tests > getClientSessions returns paginated sessions with a limit 2`] = `
 {
   "count": 4,
   "cursor": undefined,
@@ -307,7 +307,7 @@ exports[`Report storage tests > Should show paginated response of all connection
 }
 `;
 
-exports[`Report storage tests > Should show paginated response of all connections with a limit with date range 1`] = `
+exports[`Report storage tests > getClientSessions returns paginated sessions with a limit and date range 1`] = `
 {
   "count": 4,
   "cursor": "<removed-for-snapshot>",
@@ -345,7 +345,7 @@ exports[`Report storage tests > Should show paginated response of all connection
 }
 `;
 
-exports[`Report storage tests > Should show paginated response of all connections with a limit with date range 2`] = `
+exports[`Report storage tests > getClientSessions returns paginated sessions with a limit and date range 2`] = `
 {
   "count": 2,
   "cursor": undefined,
@@ -383,7 +383,7 @@ exports[`Report storage tests > Should show paginated response of all connection
 }
 `;
 
-exports[`Report storage tests > Should show paginated response of connections of specified user_id 1`] = `
+exports[`Report storage tests > getClientSessions returns sessions for a user_id 1`] = `
 {
   "count": 2,
   "cursor": undefined,
@@ -407,7 +407,7 @@ exports[`Report storage tests > Should show paginated response of connections of
 }
 `;
 
-exports[`Report storage tests > Should show paginated response of connections over a date range 1`] = `
+exports[`Report storage tests > getClientSessions returns sessions for a date range 1`] = `
 {
   "count": 6,
   "cursor": undefined,
@@ -459,7 +459,7 @@ exports[`Report storage tests > Should show paginated response of connections ov
 }
 `;
 
-exports[`Report storage tests > Should show paginated response of connections over a date range of specified client_id and user_id 1`] = `
+exports[`Report storage tests > getClientSessions returns sessions for client_id and user_id within a date range 1`] = `
 {
   "count": 1,
   "cursor": undefined,

package/test/src/__snapshots__/storage.test.ts.snap

@@ -0,0 +1,151 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Postgres Sync Bucket Storage - Data > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage - Data > (insert, delete, insert), (delete) 2`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage - Data > (insert, delete, insert), (delete) 3`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage - Data - v1 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage - Data - v2 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage - Data - v3 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;

package/test/src/__snapshots__/storage_compacting.test.ts.snap

@@ -0,0 +1,17 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Postgres Sync Bucket Storage Compact > partial checksums after compacting (2) 1`] = `
+{
+  "bucket": "1#global[]",
+  "checksum": 1196713877,
+  "count": 1,
+}
+`;
+
+exports[`Postgres Sync Bucket Storage Compact > partial checksums after compacting 1`] = `
+{
+  "bucket": "1#global[]",
+  "checksum": -134691003,
+  "count": 4,
+}
+`;