@powersync/service-module-mssql 0.0.1 → 0.1.1

package/test/src/CDCStreamTestContext.ts

@@ -12,7 +12,6 @@ import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './uti
 import { CDCStream, CDCStreamOptions } from '@module/replication/CDCStream.js';
 import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
 import timers from 'timers/promises';
-import { CDCPollingOptions } from '@module/types/types.js';
 
 /**
  * Tests operating on the change data capture need to configure the stream and manage asynchronous
@@ -110,10 +109,11 @@ export class CDCStreamTestContext implements AsyncDisposable {
       metrics: METRICS_HELPER.metricsEngine,
       connections: this.connectionManager,
       abortSignal: this.abortController.signal,
-      pollingOptions: {
-        batchSize: 10,
-        intervalMs: 1000
-      } satisfies CDCPollingOptions,
+      additionalConfig: {
+        pollingBatchSize: 10,
+        pollingIntervalMs: 1000,
+        trustServerCertificate: true
+      },
       ...this.cdcStreamOptions
     };
     this._cdcStream = new CDCStream(options);

package/test/src/CDCStream_resumable_snapshot.test.ts

@@ -1,13 +1,13 @@
 import { describe, expect, test } from 'vitest';
 import { env } from './env.js';
-import { createTestTable, createTestTableWithBasicId, describeWithStorage, waitForPendingCDCChanges } from './util.js';
+import { createTestTableWithBasicId, describeWithStorage, waitForPendingCDCChanges } from './util.js';
 import { TestStorageFactory } from '@powersync/service-core';
 import { METRICS_HELPER } from '@powersync/service-core-tests';
 import { ReplicationMetric } from '@powersync/service-types';
 import * as timers from 'node:timers/promises';
-import { ReplicationAbortedError } from '@powersync/lib-services-framework';
+import { logger, ReplicationAbortedError } from '@powersync/lib-services-framework';
 import { CDCStreamTestContext } from './CDCStreamTestContext.js';
-import { getLatestReplicatedLSN } from '@module/utils/mssql.js';
+import { getLatestLSN } from '@module/utils/mssql.js';
 
 describe.skipIf(!(env.CI || env.SLOW_TESTS))('batch replication', function () {
   describeWithStorage({ timeout: 240_000 }, function (factory) {
@@ -46,10 +46,9 @@ async function testResumingReplication(factory: TestStorageFactory, stopAfter: n
   await createTestTableWithBasicId(connectionManager, 'test_data1');
   await createTestTableWithBasicId(connectionManager, 'test_data2');
 
-  let beforeLSN = await getLatestReplicatedLSN(connectionManager);
   await connectionManager.query(`INSERT INTO test_data1(description) SELECT 'value' FROM GENERATE_SERIES(1, 1000, 1)`);
+  let beforeLSN = await getLatestLSN(connectionManager);
   await connectionManager.query(`INSERT INTO test_data2(description) SELECT 'value' FROM GENERATE_SERIES(1, 10000, 1)`);
-
   await waitForPendingCDCChanges(beforeLSN, connectionManager);
 
   const p = context.replicateSnapshot();
@@ -64,6 +63,7 @@ async function testResumingReplication(factory: TestStorageFactory, stopAfter: n
           ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0) - startRowCount;
 
         if (count >= stopAfter) {
+          logger.info(`Stopped initial replication after replicating ${count} rows.`);
           break;
         }
         await timers.setTimeout(1);
@@ -85,22 +85,28 @@ async function testResumingReplication(factory: TestStorageFactory, stopAfter: n
     cdcStreamOptions: { snapshotBatchSize: 1000 }
   });
 
-  beforeLSN = await getLatestReplicatedLSN(context2.connectionManager);
   // This delete should be using one of the ids already replicated
   const {
-    recordset: [id1]
+    recordset: [deleteResult]
   } = await context2.connectionManager.query(`DELETE TOP (1) FROM test_data2 OUTPUT DELETED.id`);
   // This update should also be using one of the ids already replicated
+  const id1 = deleteResult.id;
+  logger.info(`Deleted row with id: ${id1}`);
   const {
-    recordset: [id2]
+    recordset: [updateResult]
   } = await context2.connectionManager.query(
     `UPDATE test_data2 SET description = 'update1' OUTPUT INSERTED.id WHERE id = (SELECT TOP 1 id FROM test_data2)`
   );
+  const id2 = updateResult.id;
+  logger.info(`Updated row with id: ${id2}`);
+  beforeLSN = await getLatestLSN(context2.connectionManager);
   const {
-    recordset: [id3]
+    recordset: [insertResult]
   } = await context2.connectionManager.query(
     `INSERT INTO test_data2(description) OUTPUT INSERTED.id VALUES ('insert1')`
   );
+  const id3 = insertResult.id;
+  logger.info(`Inserted row with id: ${id3}`);
   await waitForPendingCDCChanges(beforeLSN, context2.connectionManager);
 
   await context2.loadNextSyncRules();

package/test/src/mssql-to-sqlite.test.ts

@@ -1,11 +1,13 @@
-import { SqliteInputRow } from '@powersync/service-sync-rules';
+import { SQLITE_TRUE, SqliteInputRow } from '@powersync/service-sync-rules';
 import { afterAll, beforeEach, describe, expect, test } from 'vitest';
 import { clearTestDb, createUpperCaseUUID, TEST_CONNECTION_OPTIONS, waitForPendingCDCChanges } from './util.js';
 import { CDCToSqliteRow, toSqliteInputRow } from '@module/common/mssqls-to-sqlite.js';
 import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
 import {
   enableCDCForTable,
+  escapeIdentifier,
   getCaptureInstance,
+  getLatestLSN,
   getLatestReplicatedLSN,
   getMinLSN,
   toQualifiedTableName
@@ -25,7 +27,7 @@ describe('MSSQL Data Types Tests', () => {
 
   async function setupTestTable() {
     await connectionManager.query(`
-      CREATE TABLE ${connectionManager.schema}.test_data (
+      CREATE TABLE ${escapeIdentifier(connectionManager.schema)}.test_data (
         id INT IDENTITY(1,1) PRIMARY KEY,
         tinyint_col TINYINT,
         smallint_col SMALLINT,
@@ -74,9 +76,9 @@ describe('MSSQL Data Types Tests', () => {
   }
 
   test('Number types mappings', async () => {
-    const beforeLSN = await getLatestReplicatedLSN(connectionManager);
+    const beforeLSN = await getLatestLSN(connectionManager);
     await connectionManager.query(`
-      INSERT INTO ${connectionManager.schema}.test_data(
+      INSERT INTO ${escapeIdentifier(connectionManager.schema)}.test_data(
         tinyint_col,
         smallint_col,
         int_col,
@@ -118,16 +120,16 @@ describe('MSSQL Data Types Tests', () => {
       numeric_col: 12345.67,
       money_col: 12345.67,
       smallmoney_col: 123.45,
-      bit_col: 1
+      bit_col: SQLITE_TRUE
     };
     expect(databaseRows[0]).toMatchObject(expectedResult);
     expect(replicatedRows[0]).toMatchObject(expectedResult);
   });
 
   test('Character types mappings', async () => {
-    const beforeLSN = await getLatestReplicatedLSN(connectionManager);
+    const beforeLSN = await getLatestLSN(connectionManager);
     await connectionManager.query(`
-      INSERT INTO [${connectionManager.schema}].test_data (
+      INSERT INTO ${escapeIdentifier(connectionManager.schema)}.test_data (
         char_col,
         varchar_col,
         varchar_max_col,
@@ -167,11 +169,11 @@ describe('MSSQL Data Types Tests', () => {
   });
 
   test('Binary types mappings', async () => {
-    const beforeLSN = await getLatestReplicatedLSN(connectionManager);
+    const beforeLSN = await getLatestLSN(connectionManager);
     const binaryData = Buffer.from('BinaryData');
     await connectionManager.query(
       `
-      INSERT INTO [${connectionManager.schema}].test_data (
+      INSERT INTO ${escapeIdentifier(connectionManager.schema)}.test_data (
         binary_col,
         varbinary_col,
         varbinary_max_col,
@@ -211,11 +213,11 @@ describe('MSSQL Data Types Tests', () => {
   });
 
   test('Date types mappings', async () => {
-    const beforeLSN = await getLatestReplicatedLSN(connectionManager);
+    const beforeLSN = await getLatestLSN(connectionManager);
     const testDate = new Date('2023-03-06T15:47:00.000Z');
     await connectionManager.query(
       `
-      INSERT INTO [${connectionManager.schema}].test_data(
+      INSERT INTO ${escapeIdentifier(connectionManager.schema)}.test_data(
         date_col, 
         datetime_col, 
         datetime2_col, 
@@ -255,22 +257,21 @@ describe('MSSQL Data Types Tests', () => {
   });
 
   test('Date types edge cases mappings', async () => {
-    const beforeLSN = await getLatestReplicatedLSN(connectionManager);
-
     await connectionManager.query(`
-      INSERT INTO [${connectionManager.schema}].test_data(datetime2_col) 
+      INSERT INTO ${escapeIdentifier(connectionManager.schema)}.test_data(datetime2_col) 
       VALUES ('0001-01-01 00:00:00.000')
     `);
     await connectionManager.query(`
-      INSERT INTO [${connectionManager.schema}].test_data(datetime2_col) 
+      INSERT INTO ${escapeIdentifier(connectionManager.schema)}.test_data(datetime2_col) 
       VALUES ('9999-12-31 23:59:59.999')
     `);
     await connectionManager.query(`
-      INSERT INTO [${connectionManager.schema}].test_data(datetime_col) 
+      INSERT INTO ${escapeIdentifier(connectionManager.schema)}.test_data(datetime_col) 
       VALUES ('1753-01-01 00:00:00')
     `);
+    const beforeLSN = await getLatestLSN(connectionManager);
     await connectionManager.query(`
-      INSERT INTO [${connectionManager.schema}].test_data(datetime_col) 
+      INSERT INTO ${escapeIdentifier(connectionManager.schema)}.test_data(datetime_col) 
       VALUES ('9999-12-31 23:59:59.997')
     `);
     await waitForPendingCDCChanges(beforeLSN, connectionManager);
@@ -292,10 +293,10 @@ describe('MSSQL Data Types Tests', () => {
   });
 
   test('DateTimeOffset type mapping', async () => {
-    const beforeLSN = await getLatestReplicatedLSN(connectionManager);
+    const beforeLSN = await getLatestLSN(connectionManager);
     // DateTimeOffset preserves timezone information
     await connectionManager.query(`
-      INSERT INTO [${connectionManager.schema}].test_data(datetimeoffset_col) 
+      INSERT INTO ${escapeIdentifier(connectionManager.schema)}.test_data(datetimeoffset_col) 
       VALUES ('2023-03-06 15:47:00.000 +05:00')
     `);
     await waitForPendingCDCChanges(beforeLSN, connectionManager);
@@ -313,12 +314,12 @@ describe('MSSQL Data Types Tests', () => {
   });
 
   test('UniqueIdentifier type mapping', async () => {
-    const beforeLSN = await getLatestReplicatedLSN(connectionManager);
+    const beforeLSN = await getLatestLSN(connectionManager);
 
     const testGuid = createUpperCaseUUID();
     await connectionManager.query(
       `
-      INSERT INTO [${connectionManager.schema}].test_data(uniqueidentifier_col) 
+      INSERT INTO ${escapeIdentifier(connectionManager.schema)}.test_data(uniqueidentifier_col) 
       VALUES (@guid)
     `,
       [{ name: 'guid', type: sql.UniqueIdentifier, value: testGuid }]
@@ -334,11 +335,11 @@ describe('MSSQL Data Types Tests', () => {
   });
 
   test('JSON type mapping', async () => {
-    const beforeLSN = await getLatestReplicatedLSN(connectionManager);
+    const beforeLSN = await getLatestLSN(connectionManager);
     const expectedJSON = { name: 'John Doe', age: 30, married: true };
     await connectionManager.query(
       `
-      INSERT INTO [${connectionManager.schema}].test_data(json_col) 
+      INSERT INTO ${escapeIdentifier(connectionManager.schema)}.test_data(json_col) 
       VALUES (@json)
     `,
       [{ name: 'json', type: sql.NVarChar(sql.MAX), value: JSON.stringify(expectedJSON) }]
@@ -355,11 +356,11 @@ describe('MSSQL Data Types Tests', () => {
   });
 
   test('XML type mapping', async () => {
-    const beforeLSN = await getLatestReplicatedLSN(connectionManager);
+    const beforeLSN = await getLatestLSN(connectionManager);
     const xmlData = '<root><item>value</item></root>';
     await connectionManager.query(
       `
-      INSERT INTO [${connectionManager.schema}].test_data(xml_col) 
+      INSERT INTO ${escapeIdentifier(connectionManager.schema)}.test_data(xml_col) 
       VALUES (@xml)
     `,
       [{ name: 'xml', type: sql.Xml, value: xmlData }]
@@ -375,7 +376,7 @@ describe('MSSQL Data Types Tests', () => {
 
   // TODO: Update test when properly converting spatial types
   // test('Spatial types mappings', async () => {
-  //   const beforeLSN = await getLatestReplicatedLSN(connectionManager);
+  //   const beforeLSN = await getLatestLSN(connectionManager);
   //   // Geometry and Geography types are stored as binary/WKT strings
   //   await connectionManager.query(`
   //     INSERT INTO [${connectionManager.schema}].test_data(geometry_col, geography_col)
@@ -399,7 +400,7 @@ describe('MSSQL Data Types Tests', () => {
   // TODO: Enable when HierarchyID type is properly supported
   // test('HierarchyID type mapping', async () => {
   //   const hierarchyid = '/1/';
-  //   const beforeLSN = await getLatestReplicatedLSN(connectionManager);
+  //   const beforeLSN = await getLatestLSN(connectionManager);
   //   await connectionManager.query(`
   //     INSERT INTO [${connectionManager.schema}].test_data(hierarchyid_col)
   //     VALUES (@hierarchyid)

package/test/src/util.ts

@@ -8,7 +8,7 @@ import * as postgres_storage from '@powersync/service-module-postgres-storage';
 import { describe, TestOptions } from 'vitest';
 import { env } from './env.js';
 import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
-import { createCheckpoint, enableCDCForTable, getLatestLSN } from '@module/utils/mssql.js';
+import { createCheckpoint, enableCDCForTable, escapeIdentifier, getLatestLSN } from '@module/utils/mssql.js';
 import sql from 'mssql';
 import { v4 as uuid } from 'uuid';
 import { LSN } from '@module/common/LSN.js';
@@ -37,7 +37,11 @@ export function describeWithStorage(options: TestOptions, fn: (factory: TestStor
 export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({
   type: 'mssql',
   uri: TEST_URI,
-  trustServerCertificate: true
+  additionalConfig: {
+    pollingBatchSize: 10,
+    pollingIntervalMs: 1000,
+    trustServerCertificate: true
+  }
 });
 
 /**
@@ -62,6 +66,16 @@ export async function clearTestDb(connectionManager: MSSQLConnectionManager) {
   }
 }
 
+export async function resetTestTable(connectionManager: MSSQLConnectionManager, tableName: string) {
+  await connectionManager.execute('sys.sp_cdc_disable_table', [
+    { name: 'source_schema', value: connectionManager.schema },
+    { name: 'source_name', value: tableName },
+    { name: 'capture_instance', value: 'all' }
+  ]);
+
+  await connectionManager.query(`DROP TABLE [${tableName}]`);
+}
+
 /**
  *  Create a new database for testing and enables CDC on it.
  *  @param connectionManager
@@ -80,7 +94,7 @@ export async function createTestDb(connectionManager: MSSQLConnectionManager, db
 
 export async function createTestTable(connectionManager: MSSQLConnectionManager, tableName: string): Promise<void> {
   await connectionManager.query(`
-    CREATE TABLE ${connectionManager.schema}.${tableName} (
+    CREATE TABLE ${escapeIdentifier(connectionManager.schema)}.${escapeIdentifier(tableName)} (
       id UNIQUEIDENTIFIER PRIMARY KEY,
       description VARCHAR(MAX)
     )
@@ -93,7 +107,7 @@ export async function createTestTableWithBasicId(
   tableName: string
 ): Promise<void> {
   await connectionManager.query(`
-    CREATE TABLE ${connectionManager.schema}.${tableName} (
+    CREATE TABLE ${escapeIdentifier(connectionManager.schema)}.${escapeIdentifier(tableName)} (
       id INT IDENTITY(1,1) PRIMARY KEY,
       description VARCHAR(MAX)
     )
@@ -110,7 +124,7 @@ export async function insertTestData(connectionManager: MSSQLConnectionManager,
   const description = `description_${id}`;
   await connectionManager.query(
     `
-    INSERT INTO ${connectionManager.schema}.${tableName} (id, description) VALUES (@id, @description)
+    INSERT INTO ${escapeIdentifier(connectionManager.schema)}.${escapeIdentifier(tableName)} (id, description) VALUES (@id, @description)
   `,
     [
       { name: 'id', type: sql.UniqueIdentifier, value: id },
@@ -137,8 +151,8 @@ export async function waitForPendingCDCChanges(
     );
 
     if (result.length === 0) {
-      logger.info(`CDC changes pending. Waiting for 500ms...`);
-      await new Promise((resolve) => setTimeout(resolve, 500));
+      logger.info(`CDC changes pending. Waiting for 200ms...`);
+      await new Promise((resolve) => setTimeout(resolve, 200));
     } else {
       logger.info(`Found LSN: ${LSN.fromBinary(result[0].start_lsn).toString()}`);
       return;
@@ -166,13 +180,12 @@ export async function getClientCheckpoint(
   while (Date.now() - start < timeout) {
     const storage = await storageFactory.getActiveStorage();
     const cp = await storage?.getCheckpoint();
-    if (cp == null) {
-      throw new Error('No sync rules available');
-    }
-    lastCp = cp;
-    if (cp.lsn != null && cp.lsn >= lsn.toString()) {
-      logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`);
-      return cp.checkpoint;
+    if (cp != null) {
+      lastCp = cp;
+      if (cp.lsn != null && cp.lsn >= lsn.toString()) {
+        logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`);
+        return cp.checkpoint;
+      }
     }
 
     await new Promise((resolve) => setTimeout(resolve, 30));