@powersync/service-module-postgres-storage 0.12.0 → 0.13.1

package/src/types/models/CurrentData.ts

@@ -1,5 +1,5 @@
 import * as t from 'ts-codec';
-import { hexBuffer, jsonb, pgwire_number } from '../codecs.js';
+import { bigint, hexBuffer, jsonb, pgwire_number } from '../codecs.js';
 
 export const CurrentBucket = t.object({
   bucket: t.string,
@@ -10,7 +10,7 @@ export const CurrentBucket = t.object({
 export type CurrentBucket = t.Encoded<typeof CurrentBucket>;
 export type CurrentBucketDecoded = t.Decoded<typeof CurrentBucket>;
 
-export const CurrentData = t.object({
+export const V1CurrentData = t.object({
   buckets: jsonb(t.array(CurrentBucket)),
   data: hexBuffer,
   group_id: pgwire_number,
@@ -19,5 +19,18 @@ export const CurrentData = t.object({
   source_table: t.string
 });
 
-export type CurrentData = t.Encoded<typeof CurrentData>;
-export type CurrentDataDecoded = t.Decoded<typeof CurrentData>;
+export const V3CurrentData = t.object({
+  buckets: jsonb(t.array(CurrentBucket)),
+  data: hexBuffer,
+  group_id: pgwire_number,
+  lookups: t.array(hexBuffer),
+  source_key: hexBuffer,
+  source_table: t.string,
+  pending_delete: t.Null.or(bigint)
+});
+
+export type V1CurrentData = t.Encoded<typeof V1CurrentData>;
+export type V1CurrentDataDecoded = t.Decoded<typeof V1CurrentData>;
+
+export type V3CurrentData = t.Encoded<typeof V3CurrentData>;
+export type V3CurrentDataDecoded = t.Decoded<typeof V3CurrentData>;

package/src/utils/bson.ts

@@ -6,7 +6,7 @@ import * as uuid from 'uuid';
  * JSONB columns do not directly support storing binary data which could be required in future.
  */
 
-export function replicaIdToSubkey(tableId: string, id: storage.ReplicaId): string {
+export function replicaIdToSubkey(tableId: storage.SourceTableId, id: storage.ReplicaId): string {
   // Hashed UUID from the table and id
   if (storage.isUUID(id)) {
     // Special case for UUID for backwards-compatiblity

package/src/utils/db.ts

@@ -21,6 +21,7 @@ export const dropTables = async (client: lib_postgres.DatabaseClient) => {
     await db.sql`DROP TABLE IF EXISTS instance`.execute();
     await db.sql`DROP TABLE IF EXISTS bucket_data`.execute();
     await db.sql`DROP TABLE IF EXISTS current_data`.execute();
+    await db.sql`DROP TABLE IF EXISTS v3_current_data`.execute();
     await db.sql`DROP TABLE IF EXISTS source_tables`.execute();
     await db.sql`DROP TABLE IF EXISTS write_checkpoints`.execute();
     await db.sql`DROP TABLE IF EXISTS custom_write_checkpoints`.execute();
@@ -50,6 +51,15 @@ export const truncateTables = async (db: lib_postgres.DatabaseClient) => {
         connection_report_events RESTART IDENTITY CASCADE
     `
     },
+    {
+      // TRUNCATE if v3_current_data exists
+      statement: `DO $$
+        BEGIN
+          IF to_regclass('v3_current_data') IS NOT NULL THEN
+            EXECUTE 'TRUNCATE TABLE v3_current_data RESTART IDENTITY CASCADE';
+          END IF;
+        END $$;`
+    },
     {
       statement: `ALTER SEQUENCE IF EXISTS op_id_sequence RESTART
       WITH

package/src/utils/test-utils.ts

@@ -3,6 +3,7 @@ import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js'
 import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js';
 import { PostgresReportStorage } from '../storage/PostgresReportStorage.js';
 import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js';
+import { logger as defaultLogger, createLogger, transports } from '@powersync/lib-services-framework';
 import { truncateTables } from './db.js';
 
 export type PostgresTestStorageOptions = {
@@ -32,12 +33,20 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
 
     const mockServiceContext = { configuration: { storage: BASE_CONFIG } } as unknown as ServiceContext;
 
+    // Migration logs can get really verbose in tests, so only log warnings and up.
+    const logger = createLogger({
+      level: 'warn',
+      format: defaultLogger.format,
+      transports: [new transports.Console()]
+    });
+
     if (options.down) {
       await migrationManager.migrate({
         direction: framework.migrations.Direction.Down,
         migrationContext: {
           service_context: mockServiceContext
-        }
+        },
+        logger
       });
     }
 
@@ -46,7 +55,8 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
         direction: framework.migrations.Direction.Up,
         migrationContext: {
           service_context: mockServiceContext
-        }
+        },
+        logger
       });
     }
   };
@@ -100,10 +110,7 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
         throw ex;
       }
     },
-    migrate
+    migrate,
+    tableIdStrings: true
   };
 }
-
-export function postgresTestStorageFactoryGenerator(factoryOptions: PostgresTestStorageOptions) {
-  return postgresTestSetup(factoryOptions).factory;
-}

package/test/src/__snapshots__/storage.test.ts.snap

@@ -0,0 +1,151 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Postgres Sync Bucket Storage - Data > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage - Data > (insert, delete, insert), (delete) 2`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage - Data > (insert, delete, insert), (delete) 3`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage - Data - v1 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage - Data - v2 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;
+
+exports[`Postgres Sync Bucket Storage - Data - v3 > (insert, delete, insert), (delete) 1`] = `
+[
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+  {
+    "checksum": 2871785649,
+    "object_id": "test1",
+    "op": "PUT",
+  },
+  {
+    "checksum": 2872534815,
+    "object_id": "test1",
+    "op": "REMOVE",
+  },
+]
+`;

package/test/src/__snapshots__/storage_compacting.test.ts.snap

@@ -0,0 +1,17 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Postgres Sync Bucket Storage Compact > partial checksums after compacting (2) 1`] = `
+{
+  "bucket": "1#global[]",
+  "checksum": 1196713877,
+  "count": 1,
+}
+`;
+
+exports[`Postgres Sync Bucket Storage Compact > partial checksums after compacting 1`] = `
+{
+  "bucket": "1#global[]",
+  "checksum": -134691003,
+  "count": 4,
+}
+`;