@powersync/service-module-postgres 0.17.2 → 0.18.0

package/test/src/wal_stream.test.ts

@@ -1,11 +1,10 @@
 import { MissingReplicationSlotError } from '@module/replication/WalStream.js';
-import { storage } from '@powersync/service-core';
 import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests';
 import { pgwireRows } from '@powersync/service-jpgwire';
 import { ReplicationMetric } from '@powersync/service-types';
 import * as crypto from 'crypto';
-import { afterAll, beforeAll, describe, expect, test } from 'vitest';
-import { describeWithStorage } from './util.js';
+import { describe, expect, test } from 'vitest';
+import { describeWithStorage, StorageVersionTestContext } from './util.js';
 import { WalStreamTestContext, withMaxWalSize } from './wal_stream_utils.js';
 import { JSONBig } from '@powersync/service-jsonbig';
 
@@ -20,9 +19,12 @@ describe('wal stream', () => {
   describeWithStorage({ timeout: 20_000 }, defineWalStreamTests);
 });
 
-function defineWalStreamTests(factory: storage.TestStorageFactory) {
+function defineWalStreamTests({ factory, storageVersion }: StorageVersionTestContext) {
+  const openContext = (options?: Parameters<typeof WalStreamTestContext.open>[1]) => {
+    return WalStreamTestContext.open(factory, { ...options, storageVersion });
+  };
   test('replicating basic values', async () => {
-    await using context = await WalStreamTestContext.open(factory);
+    await using context = await openContext();
     const { pool } = context;
     await context.updateSyncRules(`
 bucket_definitions:
@@ -57,7 +59,7 @@ bucket_definitions:
   });
 
   test('replicating case sensitive table', async () => {
-    await using context = await WalStreamTestContext.open(factory);
+    await using context = await openContext();
     const { pool } = context;
     await context.updateSyncRules(`
       bucket_definitions:
@@ -88,7 +90,7 @@ bucket_definitions:
   });
 
   test('replicating TOAST values', async () => {
-    await using context = await WalStreamTestContext.open(factory);
+    await using context = await openContext();
     const { pool } = context;
     await context.updateSyncRules(`
       bucket_definitions:
@@ -126,7 +128,7 @@ bucket_definitions:
   });
 
   test('replicating TRUNCATE', async () => {
-    await using context = await WalStreamTestContext.open(factory);
+    await using context = await openContext();
     const { pool } = context;
     const syncRuleContent = `
 bucket_definitions:
@@ -157,7 +159,7 @@ bucket_definitions:
   });
 
   test('replicating changing primary key', async () => {
-    await using context = await WalStreamTestContext.open(factory);
+    await using context = await openContext();
     const { pool } = context;
     await context.updateSyncRules(BASIC_SYNC_RULES);
     await pool.query(`DROP TABLE IF EXISTS test_data`);
@@ -198,7 +200,7 @@ bucket_definitions:
   });
 
   test('initial sync', async () => {
-    await using context = await WalStreamTestContext.open(factory);
+    await using context = await openContext();
     const { pool } = context;
     await context.updateSyncRules(BASIC_SYNC_RULES);
 
@@ -217,7 +219,7 @@ bucket_definitions:
   });
 
   test('record too large', async () => {
-    await using context = await WalStreamTestContext.open(factory);
+    await using context = await openContext();
     await context.updateSyncRules(`bucket_definitions:
       global:
         data:
@@ -254,7 +256,7 @@ bucket_definitions:
   });
 
   test('table not in sync rules', async () => {
-    await using context = await WalStreamTestContext.open(factory);
+    await using context = await openContext();
     const { pool } = context;
     await context.updateSyncRules(BASIC_SYNC_RULES);
 
@@ -280,7 +282,7 @@ bucket_definitions:
 
   test('reporting slot issues', async () => {
     {
-      await using context = await WalStreamTestContext.open(factory);
+      await using context = await openContext();
       const { pool } = context;
       await context.updateSyncRules(`
 bucket_definitions:
@@ -310,7 +312,7 @@ bucket_definitions:
     }
 
     {
-      await using context = await WalStreamTestContext.open(factory, { doNotClear: true });
+      await using context = await openContext({ doNotClear: true });
       const { pool } = context;
       await pool.query('DROP PUBLICATION powersync');
       await pool.query(`UPDATE test_data SET description = 'updated'`);
@@ -345,7 +347,7 @@ bucket_definitions:
 
   test('dropped replication slot', async () => {
     {
-      await using context = await WalStreamTestContext.open(factory);
+      await using context = await openContext();
       const { pool } = context;
       await context.updateSyncRules(`
 bucket_definitions:
@@ -375,7 +377,7 @@ bucket_definitions:
     }
 
     {
-      await using context = await WalStreamTestContext.open(factory, { doNotClear: true });
+      await using context = await openContext({ doNotClear: true });
       const { pool } = context;
       const storage = await context.factory.getActiveStorage();
 
@@ -396,7 +398,7 @@ bucket_definitions:
   });
 
   test('replication slot lost', async () => {
-    await using baseContext = await WalStreamTestContext.open(factory, { doNotClear: true });
+    await using baseContext = await openContext({ doNotClear: true });
 
     const serverVersion = await baseContext.connectionManager.getServerVersion();
     if (serverVersion!.compareMain('13.0.0') < 0) {
@@ -408,7 +410,7 @@ bucket_definitions:
     await using s = await withMaxWalSize(baseContext.pool, '100MB');
 
     {
-      await using context = await WalStreamTestContext.open(factory);
+      await using context = await openContext();
       const { pool } = context;
       await context.updateSyncRules(`
 bucket_definitions:
@@ -438,7 +440,7 @@ bucket_definitions:
     }
 
     {
-      await using context = await WalStreamTestContext.open(factory, { doNotClear: true });
+      await using context = await openContext({ doNotClear: true });
       const { pool } = context;
       const storage = await context.factory.getActiveStorage();
       const slotName = storage?.slot_name!;
@@ -479,7 +481,7 @@ bucket_definitions:
   });
 
   test('old date format', async () => {
-    await using context = await WalStreamTestContext.open(factory);
+    await using context = await openContext();
     await context.updateSyncRules(BASIC_SYNC_RULES);
 
     const { pool } = context;
@@ -494,7 +496,7 @@ bucket_definitions:
   });
 
   test('new date format', async () => {
-    await using context = await WalStreamTestContext.open(factory);
+    await using context = await openContext();
     await context.updateSyncRules(`
 streams:
   stream:
@@ -515,7 +517,7 @@ config:
   });
 
   test('custom types', async () => {
-    await using context = await WalStreamTestContext.open(factory);
+    await using context = await openContext();
 
     await context.updateSyncRules(`
 streams:
@@ -550,7 +552,7 @@ config:
   });
 
   test('custom types in primary key', async () => {
-    await using context = await WalStreamTestContext.open(factory);
+    await using context = await openContext();
 
     await context.updateSyncRules(`
 streams:
@@ -576,7 +578,7 @@ config:
   test('replica identity handling', async () => {
     // This specifically test a case of timestamps being used as part of the replica identity.
     // There was a regression in versions 1.15.0-1.15.5, which this tests for.
-    await using context = await WalStreamTestContext.open(factory);
+    await using context = await openContext();
     const { pool } = context;
     await context.updateSyncRules(BASIC_SYNC_RULES);
 

package/test/src/wal_stream_utils.ts

@@ -1,23 +1,27 @@
 import { PgManager } from '@module/replication/PgManager.js';
 import { PUBLICATION_NAME, WalStream, WalStreamOptions } from '@module/replication/WalStream.js';
+import { CustomTypeRegistry } from '@module/types/registry.js';
 import {
   BucketStorageFactory,
   createCoreReplicationMetrics,
   initializeCoreReplicationMetrics,
   InternalOpId,
+  LEGACY_STORAGE_VERSION,
   OplogEntry,
+  STORAGE_VERSION_CONFIG,
   storage,
-  SyncRulesBucketStorage
+  SyncRulesBucketStorage,
+  updateSyncRulesFromYaml
 } from '@powersync/service-core';
 import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
 import * as pgwire from '@powersync/service-jpgwire';
 import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js';
-import { CustomTypeRegistry } from '@module/types/registry.js';
 
 export class WalStreamTestContext implements AsyncDisposable {
   private _walStream?: WalStream;
   private abortController = new AbortController();
   private streamPromise?: Promise<void>;
+  private syncRulesId?: number;
   public storage?: SyncRulesBucketStorage;
   private replicationConnection?: pgwire.PgConnection;
   private snapshotPromise?: Promise<void>;
@@ -30,7 +34,7 @@ export class WalStreamTestContext implements AsyncDisposable {
    */
   static async open(
     factory: (options: storage.TestStorageOptions) => Promise<BucketStorageFactory>,
-    options?: { doNotClear?: boolean; walStreamOptions?: Partial<WalStreamOptions> }
+    options?: { doNotClear?: boolean; storageVersion?: number; walStreamOptions?: Partial<WalStreamOptions> }
   ) {
     const f = await factory({ doNotClear: options?.doNotClear });
     const connectionManager = new PgManager(TEST_CONNECTION_OPTIONS, {});
@@ -39,13 +43,18 @@ export class WalStreamTestContext implements AsyncDisposable {
       await clearTestDb(connectionManager.pool);
     }
 
-    return new WalStreamTestContext(f, connectionManager, options?.walStreamOptions);
+    const storageVersion = options?.storageVersion ?? LEGACY_STORAGE_VERSION;
+    const versionedBuckets = STORAGE_VERSION_CONFIG[storageVersion]?.versionedBuckets ?? false;
+
+    return new WalStreamTestContext(f, connectionManager, options?.walStreamOptions, storageVersion, versionedBuckets);
   }
 
   constructor(
     public factory: BucketStorageFactory,
     public connectionManager: PgManager,
-    private walStreamOptions?: Partial<WalStreamOptions>
+    private walStreamOptions?: Partial<WalStreamOptions>,
+    private storageVersion: number = LEGACY_STORAGE_VERSION,
+    private versionedBuckets: boolean = STORAGE_VERSION_CONFIG[storageVersion]?.versionedBuckets ?? false
   ) {
     createCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
     initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
@@ -95,7 +104,10 @@ export class WalStreamTestContext implements AsyncDisposable {
   }
 
   async updateSyncRules(content: string) {
-    const syncRules = await this.factory.updateSyncRules({ content: content, validate: true });
+    const syncRules = await this.factory.updateSyncRules(
+      updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion })
+    );
+    this.syncRulesId = syncRules.id;
     this.storage = this.factory.getInstance(syncRules);
     return this.storage!;
   }
@@ -106,6 +118,7 @@ export class WalStreamTestContext implements AsyncDisposable {
       throw new Error(`Next sync rules not available`);
     }
 
+    this.syncRulesId = syncRules.id;
     this.storage = this.factory.getInstance(syncRules);
     return this.storage!;
   }
@@ -116,6 +129,7 @@ export class WalStreamTestContext implements AsyncDisposable {
       throw new Error(`Active sync rules not available`);
     }
 
+    this.syncRulesId = syncRules.id;
     this.storage = this.factory.getInstance(syncRules);
     return this.storage!;
   }
@@ -177,9 +191,21 @@ export class WalStreamTestContext implements AsyncDisposable {
     return checkpoint;
   }
 
+  private resolveBucketName(bucket: string) {
+    if (!this.versionedBuckets || /^\d+#/.test(bucket)) {
+      return bucket;
+    }
+    if (this.syncRulesId == null) {
+      throw new Error('Sync rules not configured - call updateSyncRules() first');
+    }
+    return `${this.syncRulesId}#${bucket}`;
+  }
+
   async getBucketsDataBatch(buckets: Record<string, InternalOpId>, options?: { timeout?: number }) {
     let checkpoint = await this.getCheckpoint(options);
-    const map = new Map<string, InternalOpId>(Object.entries(buckets));
+    const map = new Map<string, InternalOpId>(
+      Object.entries(buckets).map(([bucket, opId]) => [this.resolveBucketName(bucket), opId])
+    );
     return test_utils.fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
   }
 
@@ -191,8 +217,9 @@ export class WalStreamTestContext implements AsyncDisposable {
     if (typeof start == 'string') {
       start = BigInt(start);
     }
+    const resolvedBucket = this.resolveBucketName(bucket);
     const checkpoint = await this.getCheckpoint(options);
-    const map = new Map<string, InternalOpId>([[bucket, start]]);
+    const map = new Map<string, InternalOpId>([[resolvedBucket, start]]);
     let data: OplogEntry[] = [];
     while (true) {
       const batch = this.storage!.getBucketDataBatch(checkpoint, map);
@@ -202,11 +229,29 @@ export class WalStreamTestContext implements AsyncDisposable {
       if (batches.length == 0 || !batches[0]!.chunkData.has_more) {
         break;
       }
-      map.set(bucket, BigInt(batches[0]!.chunkData.next_after));
+      map.set(resolvedBucket, BigInt(batches[0]!.chunkData.next_after));
     }
     return data;
   }
 
+  async getChecksums(buckets: string[], options?: { timeout?: number }) {
+    const checkpoint = await this.getCheckpoint(options);
+    const versionedBuckets = buckets.map((bucket) => this.resolveBucketName(bucket));
+    const checksums = await this.storage!.getChecksums(checkpoint, versionedBuckets);
+
+    const unversioned = new Map();
+    for (let i = 0; i < buckets.length; i++) {
+      unversioned.set(buckets[i], checksums.get(versionedBuckets[i])!);
+    }
+
+    return unversioned;
+  }
+
+  async getChecksum(bucket: string, options?: { timeout?: number }) {
+    const checksums = await this.getChecksums([bucket], options);
+    return checksums.get(bucket);
+  }
+
   /**
    * This does not wait for a client checkpoint.
    */
@@ -215,8 +260,9 @@ export class WalStreamTestContext implements AsyncDisposable {
     if (typeof start == 'string') {
       start = BigInt(start);
     }
+    const resolvedBucket = this.resolveBucketName(bucket);
     const { checkpoint } = await this.storage!.getCheckpoint();
-    const map = new Map<string, InternalOpId>([[bucket, start]]);
+    const map = new Map<string, InternalOpId>([[resolvedBucket, start]]);
     const batch = this.storage!.getBucketDataBatch(checkpoint, map);
     const batches = await test_utils.fromAsync(batch);
     return batches[0]?.chunkData.data ?? [];