@powersync/service-module-mssql 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/test/src/CDCPoller.test.ts

@@ -0,0 +1,382 @@
+import { LSN } from '@module/common/LSN.js';
+import { MSSQLSourceTable } from '@module/common/MSSQLSourceTable.js';
+import { CDCEventHandler, CDCPoller } from '@module/replication/CDCPoller.js';
+import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
+import {
+  createCheckpoint,
+  escapeIdentifier,
+  getCaptureInstances,
+  getLatestLSN,
+  getLatestReplicatedLSN,
+  toQualifiedTableName
+} from '@module/utils/mssql.js';
+import { getReplicationIdentityColumns } from '@module/utils/schema.js';
+import { SourceTable } from '@powersync/service-core';
+import sql from 'mssql';
+import timers from 'timers/promises';
+import { afterEach, beforeEach, describe, expect, test } from 'vitest';
+import {
+  clearTestDb,
+  createTestTableWithBasicId,
+  enableCDCForTable,
+  insertBasicIdTestData,
+  TEST_CONNECTION_OPTIONS,
+  waitForPendingCDCChanges
+} from './util.js';
+
+describe('CDCPoller tests', { timeout: 60_000 }, () => {
+  let connectionManager: MSSQLConnectionManager;
+
+  beforeEach(async () => {
+    connectionManager = new MSSQLConnectionManager(TEST_CONNECTION_OPTIONS, {});
+    await clearTestDb(connectionManager);
+  });
+
+  afterEach(async () => {
+    await connectionManager.end();
+  });
+
+  test('Deferred updates are collapsed into a single UPDATE', async () => {
+    const tableName = 'test_deferred_update';
+    await createDeferredUpdateTestTable(connectionManager, tableName);
+    const qualifiedName = toQualifiedTableName(connectionManager.schema, tableName);
+
+    const beforeInsertLSN = await getLatestLSN(connectionManager);
+    await connectionManager.query(`INSERT INTO ${qualifiedName} (id, code) VALUES (1, 'V1'), (2, 'V2')`);
+    await waitForPendingCDCChanges(beforeInsertLSN, connectionManager);
+
+    // Only replicate changes made after the initial insert.
+    const startLSN = await getLatestReplicatedLSN(connectionManager);
+
+    const beforeUpdateLSN = await getLatestLSN(connectionManager);
+    // Updating a column with a unique index, using the actual column value creates deferred updates
+    await connectionManager.query(`UPDATE ${qualifiedName} SET code = code + 'A'`);
+    await waitForPendingCDCChanges(beforeUpdateLSN, connectionManager);
+
+    const table = await resolveSourceTable(connectionManager, tableName);
+    const { operations } = await collectChanges({
+      connectionManager,
+      tables: [table],
+      startLSN,
+      expectedOperationCount: 2
+    });
+
+    expect(operations).toMatchObject([
+      { operation: 'update', rowBefore: { id: 1, code: 'V1' }, rowAfter: { id: 1, code: 'V1A' } },
+      { operation: 'update', rowBefore: { id: 2, code: 'V2' }, rowAfter: { id: 2, code: 'V2A' } }
+    ]);
+  });
+
+  test('Normal deletes and inserts in the same transaction are not collapsed into updates', async () => {
+    const tableName = 'test_mixed_transaction';
+    await createTestTableWithBasicId(connectionManager, tableName);
+    const qualifiedName = toQualifiedTableName(connectionManager.schema, tableName);
+
+    const beforeInsertLSN = await getLatestLSN(connectionManager);
+    const initial = await insertBasicIdTestData(connectionManager, tableName);
+    await waitForPendingCDCChanges(beforeInsertLSN, connectionManager);
+
+    const startLSN = await getLatestReplicatedLSN(connectionManager);
+
+    const beforeUpdateLSN = await getLatestLSN(connectionManager);
+    const replacementDescription = 'replacement';
+    const { recordset } = await connectionManager.query(
+      `
+      BEGIN TRAN;
+        DELETE FROM ${qualifiedName} WHERE id = @id;
+        INSERT INTO ${qualifiedName} (description)
+        OUTPUT INSERTED.id, INSERTED.description
+        VALUES (@description);
+      COMMIT;
+    `,
+      [
+        { name: 'id', type: sql.Int, value: initial.id },
+        { name: 'description', type: sql.NVarChar(sql.MAX), value: replacementDescription }
+      ]
+    );
+    const replacement = recordset[0];
+    await waitForPendingCDCChanges(beforeUpdateLSN, connectionManager);
+
+    const table = await resolveSourceTable(connectionManager, tableName);
+    const { operations } = await collectChanges({
+      connectionManager,
+      tables: [table],
+      startLSN,
+      expectedOperationCount: 2
+    });
+
+    expect(operations).toMatchObject([
+      { operation: 'delete', row: initial },
+      { operation: 'insert', row: replacement }
+    ]);
+  });
+
+  test('In place updates emit a single UPDATE with the before and after rows', async () => {
+    const tableName = 'test_in_place_update';
+    await createTestTableWithBasicId(connectionManager, tableName);
+    const qualifiedName = toQualifiedTableName(connectionManager.schema, tableName);
+
+    const beforeInsertLSN = await getLatestLSN(connectionManager);
+    const initial = await insertBasicIdTestData(connectionManager, tableName);
+    await waitForPendingCDCChanges(beforeInsertLSN, connectionManager);
+
+    const startLSN = await getLatestReplicatedLSN(connectionManager);
+
+    const beforeUpdateLSN = await getLatestLSN(connectionManager);
+    const updated = { ...initial, description: 'updated' };
+    await connectionManager.query(`UPDATE ${qualifiedName} SET description = @description WHERE id = @id`, [
+      { name: 'description', type: sql.NVarChar(sql.MAX), value: updated.description },
+      { name: 'id', type: sql.Int, value: updated.id }
+    ]);
+    await waitForPendingCDCChanges(beforeUpdateLSN, connectionManager);
+
+    const table = await resolveSourceTable(connectionManager, tableName);
+    const { operations } = await collectChanges({
+      connectionManager,
+      tables: [table],
+      startLSN,
+      expectedOperationCount: 1
+    });
+
+    expect(operations).toMatchObject([{ operation: 'update', rowBefore: initial, rowAfter: updated }]);
+  });
+
+  test('Committed transaction count covers transactions across all replicated tables', async () => {
+    const firstTableName = 'test_transaction_count_first';
+    const secondTableName = 'test_transaction_count_second';
+    await createTestTableWithBasicId(connectionManager, firstTableName);
+    await createTestTableWithBasicId(connectionManager, secondTableName);
+
+    const startLSN = await getLatestReplicatedLSN(connectionManager);
+
+    // Two separate transactions, each applying to only one of the tables. Previously this would have been undercounted as 1 transaction.
+    const first = await insertBasicIdTestData(connectionManager, firstTableName);
+    const second = await insertBasicIdTestData(connectionManager, secondTableName);
+
+    // CDC captures transactions in commit order, so waiting for a checkpoint written after both
+    // inserts guarantees both have been captured before polling starts. That in turn guarantees
+    // the poller sees both within a single polling cycle, which is what is being asserted here.
+    const beforeCheckpointLSN = await getLatestLSN(connectionManager);
+    await createCheckpoint(connectionManager);
+    await waitForPendingCDCChanges(beforeCheckpointLSN, connectionManager);
+
+    const { operations, commits } = await collectChanges({
+      connectionManager,
+      tables: [
+        await resolveSourceTable(connectionManager, firstTableName),
+        await resolveSourceTable(connectionManager, secondTableName)
+      ],
+      startLSN,
+      expectedOperationCount: 2
+    });
+
+    expect(operations).toMatchObject([
+      { operation: 'insert', row: first },
+      { operation: 'insert', row: second }
+    ]);
+
+    // The checkpoint table is not replicated here, so only the two inserts are counted.
+    const transactionCount = commits.reduce((total, commit) => total + commit.transactionCount, 0);
+    expect(transactionCount).toEqual(2);
+  });
+
+  test('Refreshes the bound capture instance to use its latest metadata', async () => {
+    const tableName = 'test_capture_metadata';
+    await createTestTableWithBasicId(connectionManager, tableName);
+    const table = await resolveSourceTable(connectionManager, tableName);
+    const startupInstance = table.captureInstance;
+    console.log(startupInstance);
+    const beforeInsertLSN = await getLatestLSN(connectionManager);
+    const inserted = await insertBasicIdTestData(connectionManager, tableName);
+    await waitForPendingCDCChanges(beforeInsertLSN, connectionManager);
+
+    const captureInstances = await getCaptureInstances({
+      connectionManager,
+      table: { schema: connectionManager.schema, name: tableName }
+    });
+    console.log(captureInstances.values());
+    const refreshed = captureInstances.get(table.objectId)!.instances[0];
+    const eventHandler = new RecordingCDCEventHandler();
+    const poller = new CDCPoller({
+      connectionManager,
+      eventHandler,
+      getReplicatedTables: () => [table],
+      startLSN: LSN.fromString(LSN.ZERO),
+      additionalConfig: {
+        pollingBatchSize: 1,
+        pollingIntervalMs: 1,
+        trustServerCertificate: true
+      }
+    });
+    (poller as any).captureInstances = captureInstances;
+
+    const bounds = {
+      startLSN: LSN.fromString(LSN.ZERO),
+      endLSN: await getLatestReplicatedLSN(connectionManager)
+    };
+    await (poller as any).pollTable(table, bounds);
+
+    expect(startupInstance).not.toBe(refreshed);
+    expect(bounds.startLSN).toBe(refreshed.minLSN);
+    expect(table.captureInstance).toBe(refreshed);
+    expect(table.pinnedCaptureObjectId).toBe(refreshed.objectId);
+    expect(eventHandler.operations).toMatchObject([{ operation: 'insert', row: inserted }]);
+  });
+});
+
+/**
+ *  A change as it was handed to the CDCEventHandler. The CDC rows are kept as received, so that
+ *  assertions can match on whichever source columns they care about.
+ */
+type RecordedOperation =
+  | { operation: 'insert'; row: any }
+  | { operation: 'delete'; row: any }
+  | { operation: 'update'; rowBefore: any; rowAfter: any };
+
+/**
+ *  A commit as it was reported to the CDCEventHandler at the end of a polling cycle.
+ */
+interface RecordedCommit {
+  lsn: string;
+  transactionCount: number;
+}
+
+/**
+ *  Records the operations emitted by the CDCPoller so that their relative order can be asserted.
+ */
+class RecordingCDCEventHandler implements CDCEventHandler {
+  readonly operations: RecordedOperation[] = [];
+  readonly commits: RecordedCommit[] = [];
+
+  async onInsert(row: any): Promise<void> {
+    this.operations.push({ operation: 'insert', row });
+  }
+
+  async onUpdate(rowAfter: any, rowBefore: any): Promise<void> {
+    this.operations.push({ operation: 'update', rowBefore, rowAfter });
+  }
+
+  async onDelete(row: any): Promise<void> {
+    this.operations.push({ operation: 'delete', row });
+  }
+
+  async onCommit(lsn: string, transactionCount: number): Promise<void> {
+    this.commits.push({ lsn, transactionCount });
+  }
+
+  async onSchemaChange(): Promise<void> {}
+}
+
+/**
+ *  How long to wait for the expected operations before returning whatever arrived.
+ */
+const COLLECT_CHANGES_TIMEOUT_MS = 20_000;
+
+interface CollectChangesOptions {
+  connectionManager: MSSQLConnectionManager;
+  tables: MSSQLSourceTable[];
+  startLSN: LSN;
+  expectedOperationCount: number;
+}
+
+/**
+ *  Runs a CDCPoller from startLSN until the expected number of operations has been emitted, and
+ *  returns the handler holding the operations and the commits in the order they were reported.
+ */
+async function collectChanges(options: CollectChangesOptions): Promise<RecordingCDCEventHandler> {
+  const { connectionManager, tables, startLSN, expectedOperationCount } = options;
+  const eventHandler = new RecordingCDCEventHandler();
+
+  const poller = new CDCPoller({
+    connectionManager,
+    eventHandler,
+    getReplicatedTables: () => tables,
+    startLSN,
+    additionalConfig: {
+      pollingBatchSize: 100,
+      pollingIntervalMs: 50,
+      trustServerCertificate: true
+    },
+    schemaCheckIntervalMs: 60_000
+  });
+
+  const pollerPromise = poller.replicateUntilStopped();
+  try {
+    const deadline = Date.now() + COLLECT_CHANGES_TIMEOUT_MS;
+    while (eventHandler.operations.length < expectedOperationCount && Date.now() < deadline) {
+      // Rethrows if the poller fails instead of waiting out the full timeout.
+      await Promise.race([timers.setTimeout(50), pollerPromise]);
+    }
+  } finally {
+    await poller.stop();
+    await pollerPromise;
+  }
+
+  return eventHandler;
+}
+
+/**
+ *  Creates a table with a clustered primary key and a unique index over a separate column.
+ *  A unique index is one of the prerequisites for triggering deferred updates in SQL Server.
+ */
+async function createDeferredUpdateTestTable(
+  connectionManager: MSSQLConnectionManager,
+  tableName: string
+): Promise<void> {
+  const qualifiedName = toQualifiedTableName(connectionManager.schema, tableName);
+  await connectionManager.query(`
+    CREATE TABLE ${qualifiedName} (
+      id INT NOT NULL PRIMARY KEY,
+      code VARCHAR(100) NOT NULL
+    )
+  `);
+  await connectionManager.query(
+    `CREATE UNIQUE NONCLUSTERED INDEX ${escapeIdentifier(`UX_${tableName}_code`)} ON ${qualifiedName}(code)`
+  );
+  await enableCDCForTable({ connectionManager, table: tableName });
+}
+
+/**
+ *  Builds the MSSQLSourceTable that the CDCPoller needs, including the persisted capture-instance binding
+ *  that would normally be populated by source-table reconciliation.
+ */
+async function resolveSourceTable(
+  connectionManager: MSSQLConnectionManager,
+  tableName: string
+): Promise<MSSQLSourceTable> {
+  const captureInstances = await getCaptureInstances({
+    connectionManager,
+    table: { schema: connectionManager.schema, name: tableName }
+  });
+  const details = [...captureInstances.values()][0];
+  if (details == null) {
+    throw new Error(`No CDC capture instance found for table ${tableName}`);
+  }
+
+  const replicaIdColumnsResult = await getReplicationIdentityColumns({
+    connectionManager,
+    tableName,
+    schema: connectionManager.schema
+  });
+
+  const ref = {
+    connectionTag: connectionManager.connectionTag,
+    objectId: details.sourceTable.objectId,
+    schema: details.sourceTable.schema,
+    name: details.sourceTable.name,
+    replicaIdColumns: replicaIdColumnsResult.columns
+  };
+  const sourceTable = new SourceTable({
+    id: `${details.sourceTable.objectId}`,
+    ref,
+    objectId: details.sourceTable.objectId,
+    replicaIdColumns: replicaIdColumnsResult.columns,
+    snapshotComplete: true,
+    bucketDataSources: [],
+    parameterLookupSources: [],
+    sourceMetadata: { captureTableObjectId: details.instances[0].objectId }
+  });
+  const table = new MSSQLSourceTable(ref, [sourceTable]);
+  table.setCaptureInstance(details.instances);
+  return table;
+}

package/test/src/CDCStream.test.ts

@@ -110,14 +110,16 @@ function defineCDCStreamTests(config: storage.TestStorageConfig) {
     expect(data).toMatchObject([putOp('test_data', testData), removeOp('test_data', testData.id)]);
   });
 
-  test('Replicate matched wild card tables in sync rules', async () => {
+  test('Replicate multiple tables in sync rules', async () => {
     await using context = await CDCStreamTestContext.open(factory);
     const { connectionManager } = context;
+    // Table wildcards are not supported - each table is listed explicitly.
     await context.updateSyncRules(`
   bucket_definitions:
     global:
       data:
-        - SELECT id, description FROM "test_data_%"`);
+        - SELECT id, description FROM "test_data_1"
+        - SELECT id, description FROM "test_data_2"`);
 
     await createTestTable(connectionManager, 'test_data_1');
     await createTestTable(connectionManager, 'test_data_2');
@@ -148,6 +150,8 @@ function defineCDCStreamTests(config: storage.TestStorageConfig) {
     const { connectionManager } = context;
     await context.updateSyncRules(BASIC_SYNC_RULES);
 
+    // test_data is in the sync config and must exist for replication to start; test_donotsync is not.
+    await createTestTable(connectionManager, 'test_data');
     await createTestTable(connectionManager, 'test_donotsync');
 
     await context.replicateSnapshot();

package/test/src/CDCStreamTestContext.ts

@@ -26,7 +26,7 @@ export class CDCStreamTestContext implements AsyncDisposable {
   private _cdcStream?: CDCStream;
   private abortController = new AbortController();
   private streamPromise?: Promise<void>;
-  private syncRulesContent?: storage.PersistedSyncRulesContent;
+  private syncRulesContent?: storage.PersistedSyncConfigContent;
   public storage?: SyncRulesBucketStorage;
   private snapshotPromise?: Promise<void>;
   private replicationDone = false;
@@ -74,38 +74,45 @@ export class CDCStreamTestContext implements AsyncDisposable {
     return this.connectionManager.connectionTag;
   }
 
+  /**
+   * Resolves when streaming stops, or rejects with the failure that stopped it.
+   */
+  get streamingPromise() {
+    return this.streamPromise;
+  }
+
   async updateSyncRules(content: string) {
-    const syncRules = await this.factory.updateSyncRules(
+    const replicationStream = await this.factory.updateSyncRules(
       updateSyncRulesFromYaml(content, { validate: true, storageVersion: LEGACY_STORAGE_VERSION })
     );
-    this.syncRulesContent = syncRules;
-    this.storage = this.factory.getInstance(syncRules);
+    this.syncRulesContent = replicationStream.syncConfigContent[0];
+    this.storage = this.factory.getInstance(replicationStream);
     return this.storage!;
   }
 
   async loadNextSyncRules() {
-    const syncRules = await this.factory.getNextSyncRulesContent();
-    if (syncRules == null) {
+    const syncConfig = await this.factory.getDeployingSyncConfig();
+    if (syncConfig == null) {
       throw new Error(`Next replication stream not available`);
     }
 
-    this.syncRulesContent = syncRules;
-    this.storage = this.factory.getInstance(syncRules);
+    this.syncRulesContent = syncConfig.content;
+    this.storage = syncConfig.storage;
     return this.storage!;
   }
 
   async loadActiveSyncRules() {
-    const syncRules = await this.factory.getActiveSyncRulesContent();
-    if (syncRules == null) {
+    const syncConfig = await this.factory.getActiveSyncConfig();
+    if (syncConfig == null) {
       throw new Error(`Active replication stream not available`);
     }
 
-    this.syncRulesContent = syncRules;
-    this.storage = this.factory.getInstance(syncRules);
+    this.syncRulesContent = syncConfig.content;
+    this.storage = syncConfig.storage;
     return this.storage!;
   }
 
-  private getSyncRulesContent(): storage.PersistedSyncRulesContent {
+  private getSyncConfigContent(): storage.PersistedSyncConfigContent {
     if (this.syncRulesContent == null) {
       throw new Error('Sync config not configured - call updateSyncRules() first');
     }
@@ -183,8 +190,8 @@ export class CDCStreamTestContext implements AsyncDisposable {
 
   async getBucketsDataBatch(buckets: Record<string, InternalOpId>, options?: { timeout?: number }) {
     let checkpoint = await this.getCheckpoint(options);
-    const syncRules = this.getSyncRulesContent();
-    const map = Object.entries(buckets).map(([bucket, start]) => bucketRequest(syncRules, bucket, start));
+    const syncConfigContent = this.getSyncConfigContent();
+    const map = Object.entries(buckets).map(([bucket, start]) => bucketRequest(syncConfigContent, bucket, start));
     return test_utils.fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
   }
 
@@ -196,19 +203,19 @@ export class CDCStreamTestContext implements AsyncDisposable {
     if (typeof start == 'string') {
       start = BigInt(start);
     }
-    const syncRules = this.getSyncRulesContent();
+    const syncConfigContent = this.getSyncConfigContent();
     const checkpoint = await this.getCheckpoint(options);
-    let map = [bucketRequest(syncRules, bucket, start)];
+    let map = [bucketRequest(syncConfigContent, bucket, start)];
     let data: OplogEntry[] = [];
     while (true) {
       const batch = this.storage!.getBucketDataBatch(checkpoint, map);
 
-      const batches = await test_utils.fromAsync(batch);
+      const batches = await test_utils.getBatchArray(batch);
       data = data.concat(batches[0]?.chunkData.data ?? []);
       if (batches.length == 0 || !batches[0]!.chunkData.has_more) {
         break;
       }
-      map = [bucketRequest(syncRules, bucket, BigInt(batches[0]!.chunkData.next_after))];
+      map = [bucketRequest(syncConfigContent, bucket, BigInt(batches[0]!.chunkData.next_after))];
     }
     return data;
   }
@@ -226,11 +233,11 @@ export class CDCStreamTestContext implements AsyncDisposable {
     if (typeof start == 'string') {
       start = BigInt(start);
     }
-    const { checkpoint } = await this.storage!.getCheckpoint();
-    const syncRules = this.getSyncRulesContent();
-    const map = [bucketRequest(syncRules, bucket, start)];
+    const checkpoint = await this.storage!.getCheckpoint();
+    const syncConfigContent = this.getSyncConfigContent();
+    const map = [bucketRequest(syncConfigContent, bucket, start)];
     const batch = this.storage!.getBucketDataBatch(checkpoint, map);
-    const batches = await test_utils.fromAsync(batch);
+    const batches = await test_utils.getBatchArray(batch);
     return batches[0]?.chunkData.data ?? [];
   }
 }