@powersync/service-module-mssql 0.0.1

package/src/utils/schema.ts

@@ -0,0 +1,172 @@
+import { SourceEntityDescriptor } from '@powersync/service-core';
+import { TablePattern } from '@powersync/service-sync-rules';
+import { MSSQLConnectionManager } from '../replication/MSSQLConnectionManager.js';
+import { MSSQLColumnDescriptor } from '../types/mssql-data-types.js';
+import { escapeIdentifier } from './mssql.js';
+
+export interface GetColumnsOptions {
+  connectionManager: MSSQLConnectionManager;
+  schema: string;
+  tableName: string;
+}
+
+async function getColumns(options: GetColumnsOptions): Promise<MSSQLColumnDescriptor[]> {
+  const { connectionManager, schema, tableName } = options;
+
+  const { recordset: columnResults } = await connectionManager.query(`
+      SELECT
+        col.name AS [name],
+        typ.name AS [type],
+        typ.system_type_id AS type_id,
+        typ.user_type_id AS user_type_id
+      FROM sys.columns AS col
+        JOIN sys.tables AS tbl ON tbl.object_id = col.object_id
+        JOIN sys.schemas AS sch ON sch.schema_id = tbl.schema_id
+        JOIN sys.types AS typ ON typ.user_type_id = col.user_type_id
+      WHERE sch.name = '${schema}'
+        AND tbl.name = '${tableName}'
+      ORDER BY col.column_id;
+      `);
+
+  return columnResults.map((row) => {
+    return {
+      name: row.name,
+      type: row.type,
+      typeId: row.type_id,
+      userTypeId: row.user_type_id
+    };
+  });
+}
+
+export interface GetReplicationIdentityColumnsOptions {
+  connectionManager: MSSQLConnectionManager;
+  schema: string;
+  tableName: string;
+}
+
+export interface ReplicationIdentityColumnsResult {
+  columns: MSSQLColumnDescriptor[];
+  identity: 'default' | 'nothing' | 'full' | 'index';
+}
+
+export async function getReplicationIdentityColumns(
+  options: GetReplicationIdentityColumnsOptions
+): Promise<ReplicationIdentityColumnsResult> {
+  const { connectionManager, schema, tableName } = options;
+  const { recordset: primaryKeyColumns } = await connectionManager.query(`
+      SELECT
+        col.name AS [name],
+        typ.name AS [type],
+        typ.system_type_id AS type_id,
+        typ.user_type_id AS user_type_id
+      FROM sys.tables           AS tbl
+        JOIN sys.schemas        AS sch ON sch.schema_id = tbl.schema_id
+        JOIN sys.indexes        AS idx ON idx.object_id = tbl.object_id AND idx.is_primary_key = 1
+        JOIN sys.index_columns  AS idx_col ON idx_col.object_id = idx.object_id AND idx_col.index_id = idx.index_id
+        JOIN sys.columns        AS col ON col.object_id  = idx_col.object_id AND col.column_id  = idx_col.column_id
+        JOIN sys.types          AS typ ON typ.user_type_id = col.user_type_id
+      WHERE sch.name = '${schema}'
+        AND tbl.name = '${tableName}'
+      ORDER BY idx_col.key_ordinal;
+      `);
+
+  if (primaryKeyColumns.length > 0) {
+    return {
+      columns: primaryKeyColumns.map((row) => ({
+        name: row.name,
+        type: row.type,
+        typeId: row.type_id,
+        userTypeId: row.user_type_id
+      })),
+      identity: 'default'
+    };
+  }
+
+  // No primary key, check if any of the columns have a unique constraint we can use
+  const { recordset: uniqueKeyColumns } = await connectionManager.query(`
+      SELECT
+        col.name AS [name],
+        typ.name AS [type],
+        typ.system_type_id AS type_id,
+        typ.user_type_id AS user_type_id
+      FROM sys.tables            AS tbl
+        JOIN sys.schemas         AS sch ON sch.schema_id = tbl.schema_id
+        JOIN sys.indexes         AS idx ON idx.object_id = tbl.object_id AND idx.is_unique_constraint = 1
+        JOIN sys.index_columns   AS idx_col ON idx_col.object_id = idx.object_id AND idx_col.index_id = idx.index_id
+        JOIN sys.columns         AS col ON col.object_id  = idx_col.object_id AND col.column_id  = idx_col.column_id
+        JOIN sys.types           AS typ ON typ.user_type_id = col.user_type_id
+      WHERE sch.name = '${schema}'
+        AND tbl.name = '${tableName}'
+      ORDER BY idx_col.key_ordinal;
+      `);
+
+  if (uniqueKeyColumns.length > 0) {
+    return {
+      columns: uniqueKeyColumns.map((row) => ({
+        name: row.name,
+        type: row.type,
+        typeId: row.type_id,
+        userTypeId: row.user_type_id
+      })),
+      identity: 'index'
+    };
+  }
+
+  const allColumns = await getColumns(options);
+
+  return {
+    columns: allColumns,
+    identity: 'full'
+  };
+}
+
+export type ResolvedTable = Omit<SourceEntityDescriptor, 'replicaIdColumns'>;
+
+export async function getTablesFromPattern(
+  connectionManager: MSSQLConnectionManager,
+  tablePattern: TablePattern
+): Promise<ResolvedTable[]> {
+  if (tablePattern.isWildcard) {
+    const { recordset: tableResults } = await connectionManager.query(`
+        SELECT
+          tbl.name      AS [table],
+          sch.name      AS [schema],
+          tbl.object_id AS object_id
+        FROM sys.tables tbl
+          JOIN sys.schemas sch ON tbl.schema_id = sch.schema_id
+        WHERE sch.name = '${tablePattern.schema}'
+          AND tbl.name LIKE '${tablePattern.tablePattern}'
+      `);
+
+    return tableResults
+      .map((row) => {
+        return {
+          objectId: row.object_id,
+          schema: row.schema,
+          name: row.table
+        };
+      })
+      .filter((table: ResolvedTable) => table.name.startsWith(tablePattern.tablePrefix));
+  } else {
+    const { recordset: tableResults } = await connectionManager.query(
+      `
+        SELECT
+          tbl.name      AS [table],
+          sch.name      AS [schema],
+          tbl.object_id AS object_id
+        FROM sys.tables tbl
+          JOIN sys.schemas sch ON tbl.schema_id = sch.schema_id
+        WHERE sch.name = '${tablePattern.schema}'
+          AND tbl.name = '${tablePattern.name}'
+      `
+    );
+
+    return tableResults.map((row) => {
+      return {
+        objectId: row.object_id,
+        schema: row.schema,
+        name: row.table
+      };
+    });
+  }
+}

package/test/src/CDCStream.test.ts

@@ -0,0 +1,206 @@
+import { describe, expect, test } from 'vitest';
+import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests';
+import { ReplicationMetric } from '@powersync/service-types';
+import {
+  createTestTable,
+  describeWithStorage,
+  INITIALIZED_MONGO_STORAGE_FACTORY,
+  insertTestData,
+  waitForPendingCDCChanges
+} from './util.js';
+import { storage } from '@powersync/service-core';
+import { CDCStreamTestContext } from './CDCStreamTestContext.js';
+import { getLatestReplicatedLSN } from '@module/utils/mssql.js';
+import sql from 'mssql';
+
+const BASIC_SYNC_RULES = `
+bucket_definitions:
+  global:
+    data:
+      - SELECT id, description FROM "test_data"
+`;
+
+// describe('CDCStream tests', () => {
+//   describeWithStorage({ timeout: 20_000 }, defineCDCStreamTests);
+// });
+
+defineCDCStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY);
+
+function defineCDCStreamTests(factory: storage.TestStorageFactory) {
+  test('Initial snapshot sync', async () => {
+    await using context = await CDCStreamTestContext.open(factory);
+    const { connectionManager } = context;
+    await context.updateSyncRules(BASIC_SYNC_RULES);
+
+    await createTestTable(connectionManager, 'test_data');
+    const beforeLSN = await getLatestReplicatedLSN(connectionManager);
+    const testData = await insertTestData(connectionManager, 'test_data');
+    await waitForPendingCDCChanges(beforeLSN, connectionManager);
+    const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
+
+    await context.replicateSnapshot();
+    await context.startStreaming();
+
+    const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
+    const data = await context.getBucketData('global[]');
+    expect(data).toMatchObject([putOp('test_data', testData)]);
+    expect(endRowCount - startRowCount).toEqual(1);
+  });
+
+  test('Replicate basic values', async () => {
+    await using context = await CDCStreamTestContext.open(factory);
+    const { connectionManager } = context;
+    await context.updateSyncRules(BASIC_SYNC_RULES);
+
+    await createTestTable(connectionManager, 'test_data');
+    await context.replicateSnapshot();
+
+    const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
+    const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0;
+
+    await context.startStreaming();
+
+    const testData = await insertTestData(connectionManager, 'test_data');
+
+    const data = await context.getBucketData('global[]');
+
+    expect(data).toMatchObject([putOp('test_data', testData)]);
+    const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
+    const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0;
+    expect(endRowCount - startRowCount).toEqual(1);
+    expect(endTxCount - startTxCount).toEqual(1);
+  });
+
+  test('Replicate row updates', async () => {
+    await using context = await CDCStreamTestContext.open(factory);
+    const { connectionManager } = context;
+    await context.updateSyncRules(BASIC_SYNC_RULES);
+
+    await createTestTable(connectionManager, 'test_data');
+    const beforeLSN = await getLatestReplicatedLSN(connectionManager);
+    const testData = await insertTestData(connectionManager, 'test_data');
+    await waitForPendingCDCChanges(beforeLSN, connectionManager);
+    await context.replicateSnapshot();
+
+    await context.startStreaming();
+
+    const updatedTestData = { ...testData };
+    updatedTestData.description = 'updated';
+    await connectionManager.query(`UPDATE test_data SET description = @description WHERE id = @id`, [
+      { name: 'description', type: sql.NVarChar(sql.MAX), value: updatedTestData.description },
+      { name: 'id', type: sql.UniqueIdentifier, value: updatedTestData.id }
+    ]);
+
+    const data = await context.getBucketData('global[]');
+    expect(data).toMatchObject([putOp('test_data', testData), putOp('test_data', updatedTestData)]);
+  });
+
+  test('Replicate row deletions', async () => {
+    await using context = await CDCStreamTestContext.open(factory);
+    const { connectionManager } = context;
+    await context.updateSyncRules(BASIC_SYNC_RULES);
+
+    await createTestTable(connectionManager, 'test_data');
+    const beforeLSN = await getLatestReplicatedLSN(connectionManager);
+    const testData = await insertTestData(connectionManager, 'test_data');
+    await waitForPendingCDCChanges(beforeLSN, connectionManager);
+    await context.replicateSnapshot();
+
+    await context.startStreaming();
+
+    await connectionManager.query(`DELETE FROM test_data WHERE id = @id`, [
+      { name: 'id', type: sql.UniqueIdentifier, value: testData.id }
+    ]);
+
+    const data = await context.getBucketData('global[]');
+    expect(data).toMatchObject([putOp('test_data', testData), removeOp('test_data', testData.id)]);
+  });
+
+  test('Replicate matched wild card tables in sync rules', async () => {
+    await using context = await CDCStreamTestContext.open(factory);
+    const { connectionManager } = context;
+    await context.updateSyncRules(`
+  bucket_definitions:
+    global:
+      data:
+        - SELECT id, description FROM "test_data_%"`);
+
+    await createTestTable(connectionManager, 'test_data_1');
+    await createTestTable(connectionManager, 'test_data_2');
+
+    const beforeLSN = await getLatestReplicatedLSN(connectionManager);
+    const testData11 = await insertTestData(connectionManager, 'test_data_1');
+    const testData21 = await insertTestData(connectionManager, 'test_data_2');
+    await waitForPendingCDCChanges(beforeLSN, connectionManager);
+
+    await context.replicateSnapshot();
+    await context.startStreaming();
+
+    const testData12 = await insertTestData(connectionManager, 'test_data_1');
+    const testData22 = await insertTestData(connectionManager, 'test_data_2');
+
+    const data = await context.getBucketData('global[]');
+
+    expect(data).toMatchObject([
+      putOp('test_data_1', testData11),
+      putOp('test_data_2', testData21),
+      putOp('test_data_1', testData12),
+      putOp('test_data_2', testData22)
+    ]);
+  });
+
+  test('Replication for tables not in the sync rules are ignored', async () => {
+    await using context = await CDCStreamTestContext.open(factory);
+    const { connectionManager } = context;
+    await context.updateSyncRules(BASIC_SYNC_RULES);
+
+    await createTestTable(connectionManager, 'test_donotsync');
+
+    await context.replicateSnapshot();
+
+    const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
+    const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0;
+
+    await context.startStreaming();
+
+    await insertTestData(connectionManager, 'test_donotsync');
+    const data = await context.getBucketData('global[]');
+
+    expect(data).toMatchObject([]);
+    const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
+    const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0;
+
+    // There was a transaction, but it is not counted since it is not for a table in the sync rules
+    expect(endRowCount - startRowCount).toEqual(0);
+    expect(endTxCount - startTxCount).toEqual(0);
+  });
+
+  test('Replicate case sensitive table', async () => {
+    await using context = await CDCStreamTestContext.open(factory);
+    const { connectionManager } = context;
+    await context.updateSyncRules(`
+      bucket_definitions:
+        global:
+          data:
+            - SELECT id, description FROM "test_DATA"
+      `);
+
+    await createTestTable(connectionManager, 'test_DATA');
+
+    await context.replicateSnapshot();
+
+    const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
+    const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0;
+
+    await context.startStreaming();
+
+    const testData = await insertTestData(connectionManager, 'test_DATA');
+    const data = await context.getBucketData('global[]');
+
+    expect(data).toMatchObject([putOp('test_DATA', testData)]);
+    const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
+    const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0;
+    expect(endRowCount - startRowCount).toEqual(1);
+    expect(endTxCount - startTxCount).toBeGreaterThanOrEqual(1);
+  });
+}

package/test/src/CDCStreamTestContext.ts

@@ -0,0 +1,212 @@
+import {
+  BucketStorageFactory,
+  createCoreReplicationMetrics,
+  initializeCoreReplicationMetrics,
+  InternalOpId,
+  OplogEntry,
+  storage,
+  SyncRulesBucketStorage
+} from '@powersync/service-core';
+import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
+import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js';
+import { CDCStream, CDCStreamOptions } from '@module/replication/CDCStream.js';
+import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
+import timers from 'timers/promises';
+import { CDCPollingOptions } from '@module/types/types.js';
+
+/**
+ * Tests operating on the change data capture need to configure the stream and manage asynchronous
+ * replication, which gets a little tricky.
+ *
+ * This wraps all the context required for testing, and tears it down afterward
+ * by using `await using`.
+ */
+export class CDCStreamTestContext implements AsyncDisposable {
+  private _cdcStream?: CDCStream;
+  private abortController = new AbortController();
+  private streamPromise?: Promise<void>;
+  public storage?: SyncRulesBucketStorage;
+  private snapshotPromise?: Promise<void>;
+  private replicationDone = false;
+
+  static async open(
+    factory: (options: storage.TestStorageOptions) => Promise<BucketStorageFactory>,
+    options?: { doNotClear?: boolean; cdcStreamOptions?: Partial<CDCStreamOptions> }
+  ) {
+    const f = await factory({ doNotClear: options?.doNotClear });
+    const connectionManager = new MSSQLConnectionManager(TEST_CONNECTION_OPTIONS, {});
+
+    if (!options?.doNotClear) {
+      await clearTestDb(connectionManager);
+    }
+
+    return new CDCStreamTestContext(f, connectionManager, options?.cdcStreamOptions);
+  }
+
+  constructor(
+    public factory: BucketStorageFactory,
+    public connectionManager: MSSQLConnectionManager,
+    private cdcStreamOptions?: Partial<CDCStreamOptions>
+  ) {
+    createCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
+    initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine);
+  }
+
+  async [Symbol.asyncDispose]() {
+    try {
+      await this.dispose();
+    } catch (err) {
+      console.error('Error disposing CDCStreamTestContext', err);
+    }
+  }
+
+  async dispose() {
+    this.abortController.abort();
+    await this.snapshotPromise;
+    await this.streamPromise;
+    await this.connectionManager.end();
+    await this.factory?.[Symbol.asyncDispose]();
+  }
+
+  get connectionTag() {
+    return this.connectionManager.connectionTag;
+  }
+
+  async updateSyncRules(content: string) {
+    const syncRules = await this.factory.updateSyncRules({ content: content, validate: true });
+    this.storage = this.factory.getInstance(syncRules);
+    return this.storage!;
+  }
+
+  async loadNextSyncRules() {
+    const syncRules = await this.factory.getNextSyncRulesContent();
+    if (syncRules == null) {
+      throw new Error(`Next sync rules not available`);
+    }
+
+    this.storage = this.factory.getInstance(syncRules);
+    return this.storage!;
+  }
+
+  async loadActiveSyncRules() {
+    const syncRules = await this.factory.getActiveSyncRulesContent();
+    if (syncRules == null) {
+      throw new Error(`Active sync rules not available`);
+    }
+
+    this.storage = this.factory.getInstance(syncRules);
+    return this.storage!;
+  }
+
+  get cdcStream() {
+    if (this.storage == null) {
+      throw new Error('updateSyncRules() first');
+    }
+    if (this._cdcStream) {
+      return this._cdcStream;
+    }
+    const options: CDCStreamOptions = {
+      storage: this.storage,
+      metrics: METRICS_HELPER.metricsEngine,
+      connections: this.connectionManager,
+      abortSignal: this.abortController.signal,
+      pollingOptions: {
+        batchSize: 10,
+        intervalMs: 1000
+      } satisfies CDCPollingOptions,
+      ...this.cdcStreamOptions
+    };
+    this._cdcStream = new CDCStream(options);
+    return this._cdcStream!;
+  }
+
+  /**
+   * Replicate a snapshot, start streaming, and wait for a consistent checkpoint.
+   */
+  async initializeReplication() {
+    await this.replicateSnapshot();
+    // TODO: renable this.startStreaming();
+    // Make sure we're up to date
+    await this.getCheckpoint();
+  }
+
+  async replicateSnapshot() {
+    await this.cdcStream.initReplication();
+    this.replicationDone = true;
+  }
+
+  // TODO: Enable once streaming is implemented
+  startStreaming() {
+    if (!this.replicationDone) {
+      throw new Error('Call replicateSnapshot() before startStreaming()');
+    }
+    this.streamPromise = this.cdcStream.streamChanges();
+    // Wait for the replication to start before returning.
+    // This avoids a bunch of unpredictable race conditions that appear in testing
+    return new Promise<void>(async (resolve) => {
+      while (this.cdcStream.isStartingReplication) {
+        await timers.setTimeout(50);
+      }
+
+      resolve();
+    });
+  }
+
+  async getCheckpoint(options?: { timeout?: number }) {
+    let checkpoint = await Promise.race([
+      getClientCheckpoint(this.connectionManager, this.factory, { timeout: options?.timeout ?? 15_000 }),
+      this.streamPromise
+    ]);
+    if (checkpoint == null) {
+      // This indicates an issue with the test setup - streamingPromise completed instead
+      // of getClientCheckpoint()
+      throw new Error('Test failure - streamingPromise completed');
+    }
+    return checkpoint;
+  }
+
+  async getBucketsDataBatch(buckets: Record<string, InternalOpId>, options?: { timeout?: number }) {
+    let checkpoint = await this.getCheckpoint(options);
+    const map = new Map<string, InternalOpId>(Object.entries(buckets));
+    return test_utils.fromAsync(this.storage!.getBucketDataBatch(checkpoint, map));
+  }
+
+  /**
+   * This waits for a client checkpoint.
+   */
+  async getBucketData(bucket: string, start?: InternalOpId | string | undefined, options?: { timeout?: number }) {
+    start ??= 0n;
+    if (typeof start == 'string') {
+      start = BigInt(start);
+    }
+    const checkpoint = await this.getCheckpoint(options);
+    const map = new Map<string, InternalOpId>([[bucket, start]]);
+    let data: OplogEntry[] = [];
+    while (true) {
+      const batch = this.storage!.getBucketDataBatch(checkpoint, map);
+
+      const batches = await test_utils.fromAsync(batch);
+      data = data.concat(batches[0]?.chunkData.data ?? []);
+      if (batches.length == 0 || !batches[0]!.chunkData.has_more) {
+        break;
+      }
+      map.set(bucket, BigInt(batches[0]!.chunkData.next_after));
+    }
+    return data;
+  }
+
+  /**
+   * This does not wait for a client checkpoint.
+   */
+  async getCurrentBucketData(bucket: string, start?: InternalOpId | string | undefined) {
+    start ??= 0n;
+    if (typeof start == 'string') {
+      start = BigInt(start);
+    }
+    const { checkpoint } = await this.storage!.getCheckpoint();
+    const map = new Map<string, InternalOpId>([[bucket, start]]);
+    const batch = this.storage!.getBucketDataBatch(checkpoint, map);
+    const batches = await test_utils.fromAsync(batch);
+    return batches[0]?.chunkData.data ?? [];
+  }
+}