@powersync/service-module-mssql 0.0.0-dev-20260225160713 → 0.0.0-dev-20260511080634

package/test/src/CDCStreamTestContext.ts

@@ -1,3 +1,5 @@
+import { CDCStream, CDCStreamOptions } from '@module/replication/CDCStream.js';
+import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
 import {
   BucketStorageFactory,
   createCoreReplicationMetrics,
@@ -9,11 +11,9 @@ import {
   SyncRulesBucketStorage,
   updateSyncRulesFromYaml
 } from '@powersync/service-core';
-import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
-import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js';
-import { CDCStream, CDCStreamOptions } from '@module/replication/CDCStream.js';
-import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
+import { bucketRequest, METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
 import timers from 'timers/promises';
+import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js';
 
 /**
  * Tests operating on the change data capture need to configure the stream and manage asynchronous
@@ -26,6 +26,7 @@ export class CDCStreamTestContext implements AsyncDisposable {
   private _cdcStream?: CDCStream;
   private abortController = new AbortController();
   private streamPromise?: Promise<void>;
+  private syncRulesContent?: storage.PersistedSyncRulesContent;
   public storage?: SyncRulesBucketStorage;
   private snapshotPromise?: Promise<void>;
   private replicationDone = false;
@@ -77,6 +78,7 @@ export class CDCStreamTestContext implements AsyncDisposable {
     const syncRules = await this.factory.updateSyncRules(
       updateSyncRulesFromYaml(content, { validate: true, storageVersion: LEGACY_STORAGE_VERSION })
     );
+    this.syncRulesContent = syncRules;
     this.storage = this.factory.getInstance(syncRules);
     return this.storage!;
   }
@@ -84,9 +86,10 @@ export class CDCStreamTestContext implements AsyncDisposable {
   async loadNextSyncRules() {
     const syncRules = await this.factory.getNextSyncRulesContent();
     if (syncRules == null) {
-      throw new Error(`Next sync rules not available`);
+      throw new Error(`Next replication stream not available`);
     }
 
+    this.syncRulesContent = syncRules;
     this.storage = this.factory.getInstance(syncRules);
     return this.storage!;
   }
@@ -94,13 +97,21 @@ export class CDCStreamTestContext implements AsyncDisposable {
   async loadActiveSyncRules() {
     const syncRules = await this.factory.getActiveSyncRulesContent();
     if (syncRules == null) {
-      throw new Error(`Active sync rules not available`);
+      throw new Error(`Active replication stream not available`);
     }
 
+    this.syncRulesContent = syncRules;
     this.storage = this.factory.getInstance(syncRules);
     return this.storage!;
   }
 
+  private getSyncRulesContent(): storage.PersistedSyncRulesContent {
+    if (this.syncRulesContent == null) {
+      throw new Error('Sync config not configured - call updateSyncRules() first');
+    }
+    return this.syncRulesContent;
+  }
+
   get cdcStream() {
     if (this.storage == null) {
       throw new Error('updateSyncRules() first');
@@ -118,6 +129,7 @@ export class CDCStreamTestContext implements AsyncDisposable {
         pollingIntervalMs: 1000,
         trustServerCertificate: true
       },
+      schemaCheckIntervalMs: 500,
       ...this.cdcStreamOptions
     };
     this._cdcStream = new CDCStream(options);
@@ -129,7 +141,7 @@ export class CDCStreamTestContext implements AsyncDisposable {
    */
   async initializeReplication() {
     await this.replicateSnapshot();
-    // TODO: renable this.startStreaming();
+    await this.startStreaming();
     // Make sure we're up to date
     await this.getCheckpoint();
   }
@@ -139,11 +151,11 @@ export class CDCStreamTestContext implements AsyncDisposable {
     this.replicationDone = true;
   }
 
-  // TODO: Enable once streaming is implemented
   startStreaming() {
     if (!this.replicationDone) {
       throw new Error('Call replicateSnapshot() before startStreaming()');
     }
+
     this.streamPromise = this.cdcStream.streamChanges();
     // Wait for the replication to start before returning.
     // This avoids a bunch of unpredictable race conditions that appear in testing
@@ -171,7 +183,8 @@ export class CDCStreamTestContext implements AsyncDisposable {
 
   async getBucketsDataBatch(buckets: Record<string, InternalOpId>, options?: { timeout?: number }) {
     let checkpoint = await this.getCheckpoint(options);
-    const map = new Map<string, InternalOpId>(Object.entries(buckets));
+    const syncRules = this.getSyncRulesContent();
+    const map = Object.entries(buckets).map(([bucket, start]) => bucketRequest(syncRules, bucket, start));
     return test_utils.fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
   }
 
@@ -183,8 +196,9 @@ export class CDCStreamTestContext implements AsyncDisposable {
     if (typeof start == 'string') {
       start = BigInt(start);
     }
+    const syncRules = this.getSyncRulesContent();
     const checkpoint = await this.getCheckpoint(options);
-    const map = new Map<string, InternalOpId>([[bucket, start]]);
+    let map = [bucketRequest(syncRules, bucket, start)];
     let data: OplogEntry[] = [];
     while (true) {
       const batch = this.storage!.getBucketDataBatch(checkpoint, map);
@@ -194,11 +208,16 @@ export class CDCStreamTestContext implements AsyncDisposable {
       if (batches.length == 0 || !batches[0]!.chunkData.has_more) {
         break;
       }
-      map.set(bucket, BigInt(batches[0]!.chunkData.next_after));
+      map = [bucketRequest(syncRules, bucket, BigInt(batches[0]!.chunkData.next_after))];
     }
     return data;
   }
 
+  async getFinalBucketState(bucket: string) {
+    const data = await this.getBucketData(bucket);
+    return test_utils.reduceBucket(data).slice(1);
+  }
+
   /**
    * This does not wait for a client checkpoint.
    */
@@ -208,7 +227,8 @@ export class CDCStreamTestContext implements AsyncDisposable {
       start = BigInt(start);
     }
     const { checkpoint } = await this.storage!.getCheckpoint();
-    const map = new Map<string, InternalOpId>([[bucket, start]]);
+    const syncRules = this.getSyncRulesContent();
+    const map = [bucketRequest(syncRules, bucket, start)];
     const batch = this.storage!.getBucketDataBatch(checkpoint, map);
     const batches = await test_utils.fromAsync(batch);
     return batches[0]?.chunkData.data ?? [];

package/test/src/CDCStream_resumable_snapshot.test.ts

@@ -1,28 +1,28 @@
-import { describe, expect, test } from 'vitest';
-import { env } from './env.js';
-import { createTestTableWithBasicId, describeWithStorage, waitForPendingCDCChanges } from './util.js';
-import { TestStorageFactory } from '@powersync/service-core';
+import { getLatestLSN } from '@module/utils/mssql.js';
+import { logger, ReplicationAbortedError } from '@powersync/lib-services-framework';
+import { TestStorageConfig } from '@powersync/service-core';
 import { METRICS_HELPER } from '@powersync/service-core-tests';
 import { ReplicationMetric } from '@powersync/service-types';
 import * as timers from 'node:timers/promises';
-import { logger, ReplicationAbortedError } from '@powersync/lib-services-framework';
+import { describe, expect, test } from 'vitest';
 import { CDCStreamTestContext } from './CDCStreamTestContext.js';
-import { getLatestLSN } from '@module/utils/mssql.js';
+import { env } from './env.js';
+import { createTestTableWithBasicId, describeWithStorage, waitForPendingCDCChanges } from './util.js';
 
 describe.skipIf(!(env.CI || env.SLOW_TESTS))('batch replication', function () {
-  describeWithStorage({ timeout: 240_000 }, function (factory) {
+  describeWithStorage({ timeout: 240_000 }, function (config) {
     test('resuming initial replication (1)', async () => {
       // Stop early - likely to not include deleted row in first replication attempt.
-      await testResumingReplication(factory, 2000);
+      await testResumingReplication(config, 2000);
     });
     test('resuming initial replication (2)', async () => {
       // Stop late - likely to include deleted row in first replication attempt.
-      await testResumingReplication(factory, 8000);
+      await testResumingReplication(config, 8000);
     });
   });
 });
 
-async function testResumingReplication(factory: TestStorageFactory, stopAfter: number) {
+async function testResumingReplication(config: TestStorageConfig, stopAfter: number) {
   // This tests interrupting and then resuming initial replication.
   // We interrupt replication after test_data1 has fully replicated, and
   // test_data2 has partially replicated.
@@ -34,7 +34,9 @@ async function testResumingReplication(factory: TestStorageFactory, stopAfter: n
   //    have been / have not been replicated at that point is not deterministic.
   //    We do allow for some variation in the test results to account for this.
 
-  await using context = await CDCStreamTestContext.open(factory, { cdcStreamOptions: { snapshotBatchSize: 1000 } });
+  await using context = await CDCStreamTestContext.open(config.factory, {
+    cdcStreamOptions: { snapshotBatchSize: 1000 }
+  });
 
   await context.updateSyncRules(`bucket_definitions:
   global:
@@ -84,7 +86,7 @@ async function testResumingReplication(factory: TestStorageFactory, stopAfter: n
   }
 
   // Bypass the usual "clear db on factory open" step.
-  await using context2 = await CDCStreamTestContext.open(factory, {
+  await using context2 = await CDCStreamTestContext.open(config.factory, {
     doNotClear: true,
     cdcStreamOptions: { snapshotBatchSize: 1000 }
   });

package/test/src/LSN.test.ts

@@ -1,5 +1,5 @@
-import { describe, expect, test } from 'vitest';
 import { LSN } from '@module/common/LSN.js';
+import { describe, expect, test } from 'vitest';
 
 describe('LSN', () => {
   test('normalizes lowercase hex strings to uppercase', () => {

package/test/src/env.ts

@@ -5,7 +5,7 @@ export const env = utils.collectEnvironmentVariables({
   MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'),
   CI: utils.type.boolean.default('false'),
   SLOW_TESTS: utils.type.boolean.default('false'),
-  PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5431/powersync_storage_test'),
+  PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_storage_test'),
   TEST_MONGO_STORAGE: utils.type.boolean.default('true'),
   TEST_POSTGRES_STORAGE: utils.type.boolean.default('true')
 });

package/test/src/mssql-to-sqlite.test.ts

@@ -1,3 +1,13 @@
+import { CDCToSqliteRow, toSqliteInputRow } from '@module/common/mssqls-to-sqlite.js';
+import { CDC_SCHEMA } from '@module/common/MSSQLSourceTable.js';
+import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
+import {
+  escapeIdentifier,
+  getCaptureInstances,
+  getLatestLSN,
+  getLatestReplicatedLSN,
+  toQualifiedTableName
+} from '@module/utils/mssql.js';
 import {
   applyRowContext,
   CompatibilityContext,
@@ -5,20 +15,15 @@ import {
   SqliteInputRow,
   TimeValuePrecision
 } from '@powersync/service-sync-rules';
+import sql from 'mssql';
 import { afterAll, beforeEach, describe, expect, test } from 'vitest';
-import { clearTestDb, createUpperCaseUUID, TEST_CONNECTION_OPTIONS, waitForPendingCDCChanges } from './util.js';
-import { CDCToSqliteRow, toSqliteInputRow } from '@module/common/mssqls-to-sqlite.js';
-import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
 import {
+  clearTestDb,
+  createUpperCaseUUID,
   enableCDCForTable,
-  escapeIdentifier,
-  getCaptureInstance,
-  getLatestLSN,
-  getLatestReplicatedLSN,
-  getMinLSN,
-  toQualifiedTableName
-} from '@module/utils/mssql.js';
-import sql from 'mssql';
+  TEST_CONNECTION_OPTIONS,
+  waitForPendingCDCChanges
+} from './util.js';
 
 describe('MSSQL Data Types Tests', () => {
   const connectionManager = new MSSQLConnectionManager(TEST_CONNECTION_OPTIONS, {});
@@ -482,20 +487,23 @@ async function getReplicatedRows(
 ): Promise<SqliteInputRow[]> {
   const endLSN = await getLatestReplicatedLSN(connectionManager);
 
-  const captureInstance = await getCaptureInstance({
+  const captureInstances = await getCaptureInstances({
     connectionManager,
-    schema: connectionManager.schema,
-    tableName
+    table: {
+      schema: connectionManager.schema,
+      name: tableName
+    }
   });
-  if (!captureInstance) {
+  if (captureInstances.size === 0) {
     throw new Error(`No CDC capture instance found for table ${tableName}`);
   }
 
-  const startLSN = await getMinLSN(connectionManager, captureInstance.name);
+  const captureInstance = Array.from(captureInstances.values())[0].instances[0];
+  const startLSN = captureInstance.minLSN;
   // Query CDC changes
   const { recordset: results } = await connectionManager.query(
     `
-      SELECT * FROM ${captureInstance.schema}.fn_cdc_get_all_changes_${captureInstance.name}(@from_lsn, @to_lsn, 'all update old') ORDER BY __$start_lsn, __$seqval
+      SELECT * FROM ${CDC_SCHEMA}.fn_cdc_get_all_changes_${captureInstance.name}(@from_lsn, @to_lsn, 'all update old') ORDER BY __$start_lsn, __$seqval
   `,
     [
       { name: 'from_lsn', type: sql.VarBinary, value: startLSN.toBinary() },