@powersync/service-module-mssql 0.0.1

package/src/api/MSSQLRouteAPIAdapter.ts

@@ -0,0 +1,283 @@
+import {
+  api,
+  ParseSyncRulesOptions,
+  PatternResult,
+  ReplicationHeadCallback,
+  ReplicationLagOptions
+} from '@powersync/service-core';
+import * as service_types from '@powersync/service-types';
+import * as sync_rules from '@powersync/service-sync-rules';
+import { SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
+import * as types from '../types/types.js';
+import { ExecuteSqlResponse } from '@powersync/service-types/dist/routes.js';
+import { MSSQLConnectionManager } from '../replication/MSSQLConnectionManager.js';
+import {
+  checkSourceConfiguration,
+  createCheckpoint,
+  getDebugTableInfo,
+  getLatestLSN,
+  POWERSYNC_CHECKPOINTS_TABLE
+} from '../utils/mssql.js';
+import { getTablesFromPattern, ResolvedTable } from '../utils/schema.js';
+import { toExpressionTypeFromMSSQLType } from '../common/mssqls-to-sqlite.js';
+
+export class MSSQLRouteAPIAdapter implements api.RouteAPI {
+  protected connectionManager: MSSQLConnectionManager;
+
+  constructor(protected config: types.ResolvedMSSQLConnectionConfig) {
+    this.connectionManager = new MSSQLConnectionManager(config, {});
+  }
+
+  async createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T> {
+    const currentLSN = await getLatestLSN(this.connectionManager);
+    const result = await callback(currentLSN.toString());
+
+    // Updates the powersync checkpoints table on the source database, ensuring that an update with a newer LSN will be captured by the CDC.
+    await createCheckpoint(this.connectionManager);
+
+    return result;
+  }
+
+  async executeQuery(query: string, params: any[]): Promise<ExecuteSqlResponse> {
+    if (!this.config.debug_api) {
+      return service_types.internal_routes.ExecuteSqlResponse.encode({
+        results: {
+          columns: [],
+          rows: []
+        },
+        success: false,
+        error: 'SQL querying is not enabled'
+      });
+    }
+    try {
+      const { recordset: result } = await this.connectionManager.query(query, params);
+      return service_types.internal_routes.ExecuteSqlResponse.encode({
+        success: true,
+        results: {
+          columns: Object.values(result.columns).map((column) => column.name),
+          rows: result.map((row) => {
+            return Object.values(row).map((value: any) => {
+              const sqlValue = sync_rules.applyValueContext(
+                sync_rules.toSyncRulesValue(row),
+                sync_rules.CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY
+              );
+
+              if (typeof sqlValue == 'bigint') {
+                return Number(row);
+              } else if (value instanceof Date) {
+                return value.toISOString();
+              } else if (sync_rules.isJsonValue(sqlValue)) {
+                return sqlValue;
+              } else {
+                return null;
+              }
+            });
+          })
+        }
+      });
+    } catch (e) {
+      return service_types.internal_routes.ExecuteSqlResponse.encode({
+        results: {
+          columns: [],
+          rows: []
+        },
+        success: false,
+        error: e.message
+      });
+    }
+  }
+
+  async getConnectionSchema(): Promise<service_types.DatabaseSchema[]> {
+    const { recordset: results } = await this.connectionManager.query(`
+      SELECT 
+        sch.name AS schema_name,
+        tbl.name AS table_name,
+        col.name AS column_name,
+        typ.name AS data_type,
+        CASE 
+          WHEN typ.name IN ('nvarchar', 'nchar') 
+            AND col.max_length > 0 
+            AND col.max_length != -1
+          THEN typ.name + '(' + CAST(col.max_length / 2 AS VARCHAR) + ')'
+          WHEN typ.name IN ('varchar', 'char', 'varbinary', 'binary') 
+            AND col.max_length > 0 
+            AND col.max_length != -1
+          THEN typ.name + '(' + CAST(col.max_length AS VARCHAR) + ')'
+          WHEN typ.name IN ('varchar', 'nvarchar', 'char', 'nchar') 
+            AND col.max_length = -1
+          THEN typ.name + '(MAX)'
+          WHEN typ.name IN ('decimal', 'numeric')
+            AND col.precision > 0
+          THEN typ.name + '(' + CAST(col.precision AS VARCHAR) + ',' + CAST(col.scale AS VARCHAR) + ')'
+          WHEN typ.name IN ('float', 'real')
+            AND col.precision > 0
+          THEN typ.name + '(' + CAST(col.precision AS VARCHAR) + ')'
+          ELSE typ.name
+        END AS formatted_type
+      FROM sys.tables AS tbl
+        JOIN sys.schemas AS sch ON sch.schema_id = tbl.schema_id
+        JOIN sys.columns AS col ON col.object_id = tbl.object_id
+        JOIN sys.types AS typ ON typ.user_type_id = col.user_type_id
+      WHERE sch.name = '${this.connectionManager.schema}'
+        AND sch.name NOT IN ('sys', 'INFORMATION_SCHEMA', 'cdc')
+        AND tbl.name NOT IN ('systranschemas', '${POWERSYNC_CHECKPOINTS_TABLE}')
+        AND tbl.type = 'U'
+        AND col.is_computed = 0
+      ORDER BY sch.name, tbl.name, col.column_id
+    `);
+
+    /**
+     * Reduces the SQL results into a Record of {@link DatabaseSchema}
+     * then returns the values as an array.
+     */
+    const schemas: Record<string, service_types.DatabaseSchema> = {};
+
+    for (const row of results) {
+      const schemaName = row.schema_name as string;
+      const tableName = row.table_name as string;
+      const columnName = row.column_name as string;
+      const dataType = row.data_type as string;
+      const formattedType = (row.formatted_type as string) || dataType;
+
+      const schema =
+        schemas[schemaName] ||
+        (schemas[schemaName] = {
+          name: schemaName,
+          tables: []
+        });
+
+      let table = schema.tables.find((t) => t.name === tableName);
+      if (!table) {
+        table = {
+          name: tableName,
+          columns: []
+        };
+        schema.tables.push(table);
+      }
+
+      table.columns.push({
+        name: columnName,
+        type: formattedType,
+        sqlite_type: toExpressionTypeFromMSSQLType(dataType).typeFlags,
+        internal_type: formattedType,
+        pg_type: formattedType
+      });
+    }
+
+    return Object.values(schemas);
+  }
+
+  async getConnectionStatus(): Promise<service_types.ConnectionStatusV2> {
+    const base = {
+      id: this.config?.id ?? '',
+      uri: this.config == null ? '' : types.baseUri(this.config)
+    };
+
+    try {
+      await this.connectionManager.query(`SELECT 'PowerSync connection test'`);
+    } catch (e) {
+      return {
+        ...base,
+        connected: false,
+        errors: [{ level: 'fatal', message: `${e.code} - message: ${e.message}` }]
+      };
+    }
+
+    try {
+      const errors = await checkSourceConfiguration(this.connectionManager);
+      if (errors.length) {
+        return {
+          ...base,
+          connected: true,
+          errors: errors.map((e) => ({ level: 'fatal', message: e }))
+        };
+      }
+    } catch (e) {
+      return {
+        ...base,
+        connected: true,
+        errors: [{ level: 'fatal', message: e.message }]
+      };
+    }
+
+    return {
+      ...base,
+      connected: true,
+      errors: []
+    };
+  }
+
+  async getDebugTablesInfo(tablePatterns: TablePattern[], sqlSyncRules: SqlSyncRules): Promise<PatternResult[]> {
+    const result: PatternResult[] = [];
+
+    for (const tablePattern of tablePatterns) {
+      const schema = tablePattern.schema;
+      const patternResult: PatternResult = {
+        schema: schema,
+        pattern: tablePattern.tablePattern,
+        wildcard: tablePattern.isWildcard
+      };
+      result.push(patternResult);
+
+      const tables = await getTablesFromPattern(this.connectionManager, tablePattern);
+      if (tablePattern.isWildcard) {
+        patternResult.tables = [];
+        for (const table of tables) {
+          const details = await getDebugTableInfo({
+            connectionManager: this.connectionManager,
+            tablePattern,
+            table,
+            syncRules: sqlSyncRules
+          });
+          patternResult.tables.push(details);
+        }
+      } else {
+        if (tables.length == 0) {
+          // This should tenchnically never happen, but we'll handle it anyway.
+          const resolvedTable: ResolvedTable = {
+            objectId: 0,
+            schema: schema,
+            name: tablePattern.name
+          };
+          patternResult.table = await getDebugTableInfo({
+            connectionManager: this.connectionManager,
+            tablePattern,
+            table: resolvedTable,
+            syncRules: sqlSyncRules
+          });
+        } else {
+          patternResult.table = await getDebugTableInfo({
+            connectionManager: this.connectionManager,
+            tablePattern,
+            table: tables[0],
+            syncRules: sqlSyncRules
+          });
+        }
+      }
+    }
+
+    return result;
+  }
+
+  getParseSyncRulesOptions(): ParseSyncRulesOptions {
+    return {
+      defaultSchema: this.connectionManager.schema
+    };
+  }
+
+  async getReplicationLagBytes(options: ReplicationLagOptions): Promise<number | undefined> {
+    return undefined;
+  }
+
+  async getSourceConfig(): Promise<service_types.configFile.ResolvedDataSourceConfig> {
+    return this.config;
+  }
+
+  async [Symbol.asyncDispose]() {
+    await this.shutdown();
+  }
+
+  async shutdown(): Promise<void> {
+    await this.connectionManager.end();
+  }
+}

package/src/common/LSN.ts

@@ -0,0 +1,77 @@
+import { ReplicationAssertionError } from '@powersync/service-errors';
+
+/**
+ *  Helper class for interpreting and manipulating SQL Server Log Sequence Numbers (LSNs).
+ *  In SQL Server, an LSN is stored as a 10-byte binary value.
+ *  But it is commonly represented in a human-readable format as three hexadecimal parts separated by colons:
+ *  `00000000:00000000:0000`.
+ *
+ *  The three parts represent different hierarchical levels of the transaction log:
+ *  1. The first part identifies the Virtual Log File (VLF).
+ *  2. The second part points to the log block within the VLF.
+ *  3. The third part specifies the exact log record within the log block.
+ */
+
+export class LSN {
+  /**
+   *  The zero or null LSN value. All other LSN values are greater than this.
+   */
+  static ZERO = '00000000:00000000:0000';
+
+  protected value: string;
+
+  private constructor(lsn: string) {
+    this.value = lsn;
+  }
+
+  /**
+   *  Converts this LSN back into its raw 10-byte binary representation for use in SQL Server functions.
+   */
+  toBinary(): Buffer {
+    let sanitized: string = this.value.replace(/:/g, '');
+    return Buffer.from(sanitized, 'hex');
+  }
+
+  /**
+   *  Converts a raw 10-byte binary LSN value into its string representation.
+   *  An error is thrown if the binary value is not exactly 10 bytes.
+   *  @param rawLSN
+   */
+  static fromBinary(rawLSN: Buffer): LSN {
+    if (rawLSN.length !== 10) {
+      throw new ReplicationAssertionError(`LSN must be 10 bytes, got ${rawLSN.length}`);
+    }
+    const hex = rawLSN.toString('hex').toUpperCase(); // 20 hex chars
+
+    return new LSN(`${hex.slice(0, 8)}:${hex.slice(8, 16)}:${hex.slice(16, 20)}`);
+  }
+
+  /**
+   *  Creates an LSN instance from the provided string representation. An error is thrown if the format is invalid.
+   *  @param stringLSN
+   */
+  static fromString(stringLSN: string): LSN {
+    if (!/^[0-9A-Fa-f]{8}:[0-9A-Fa-f]{8}:[0-9A-Fa-f]{4}$/.test(stringLSN)) {
+      throw new ReplicationAssertionError(
+        `Invalid LSN string. Expected format is [00000000:00000000:0000]. Got: ${stringLSN}`
+      );
+    }
+
+    return new LSN(stringLSN);
+  }
+
+  compare(other: LSN): -1 | 0 | 1 {
+    if (this.value === other.value) {
+      return 0;
+    }
+    return this.value < other.value ? -1 : 1;
+  }
+
+  valueOf(): string {
+    return this.value;
+  }
+
+  toString(): string {
+    return this.value;
+  }
+}

package/src/common/MSSQLSourceTable.ts

@@ -0,0 +1,54 @@
+import { SourceTable } from '@powersync/service-core';
+import { toQualifiedTableName } from '../utils/mssql.js';
+
+export interface CaptureInstance {
+  name: string;
+  schema: string;
+}
+
+export interface MSSQLSourceTableOptions {
+  sourceTable: SourceTable;
+  /**
+   *  The unique name of the CDC capture instance for this table
+   */
+  captureInstance: CaptureInstance;
+}
+
+export class MSSQLSourceTable {
+  constructor(private options: MSSQLSourceTableOptions) {}
+
+  get sourceTable() {
+    return this.options.sourceTable;
+  }
+
+  updateSourceTable(updated: SourceTable): void {
+    this.options.sourceTable = updated;
+  }
+
+  get captureInstance() {
+    return this.options.captureInstance.name;
+  }
+
+  get cdcSchema() {
+    return this.options.captureInstance.schema;
+  }
+
+  get CTTable() {
+    return `${this.cdcSchema}.${this.captureInstance}_CT`;
+  }
+
+  get allChangesFunction() {
+    return `${this.cdcSchema}.fn_cdc_get_all_changes_${this.captureInstance}`;
+  }
+
+  get netChangesFunction() {
+    return `${this.cdcSchema}.fn_cdc_get_net_changes_${this.captureInstance}`;
+  }
+
+  /**
+   *  Escapes this source table's name and schema for use in MSSQL queries.
+   */
+  toQualifiedName(): string {
+    return toQualifiedTableName(this.sourceTable.schema, this.sourceTable.name);
+  }
+}

package/src/common/MSSQLSourceTableCache.ts

@@ -0,0 +1,36 @@
+import { SourceTable } from '@powersync/service-core';
+import { MSSQLSourceTable } from './MSSQLSourceTable.js';
+import { ServiceAssertionError } from '@powersync/service-errors';
+
+export class MSSQLSourceTableCache {
+  private cache = new Map<number | string, MSSQLSourceTable>();
+
+  set(table: MSSQLSourceTable): void {
+    this.cache.set(table.sourceTable.objectId!, table);
+  }
+
+  /**
+   *  Updates the underlying source table of the cached MSSQLSourceTable.
+   *  @param updatedTable
+   */
+  updateSourceTable(updatedTable: SourceTable) {
+    const existingTable = this.cache.get(updatedTable.objectId!);
+
+    if (!existingTable) {
+      throw new ServiceAssertionError('Tried to update a non-existing MSSQLSourceTable in the cache');
+    }
+    existingTable.updateSourceTable(updatedTable);
+  }
+
+  get(tableId: number): MSSQLSourceTable | undefined {
+    return this.cache.get(tableId);
+  }
+
+  getAll(): MSSQLSourceTable[] {
+    return Array.from(this.cache.values());
+  }
+
+  delete(tableId: number): boolean {
+    return this.cache.delete(tableId);
+  }
+}

package/src/common/mssqls-to-sqlite.ts

@@ -0,0 +1,151 @@
+import sql from 'mssql';
+import { DatabaseInputRow, ExpressionType, SqliteInputRow, toSyncRulesRow } from '@powersync/service-sync-rules';
+import { MSSQLUserDefinedType } from '../types/mssql-data-types.js';
+
+export function toSqliteInputRow(row: any, columns: sql.IColumnMetadata): SqliteInputRow {
+  let result: DatabaseInputRow = {};
+  for (const key in row) {
+    // We are very much expecting the column to be there
+    const columnMetadata = columns[key];
+
+    if (row[key] !== null) {
+      switch (columnMetadata.type) {
+        case sql.TYPES.BigInt:
+          // MSSQL returns BIGINT as a string to avoid precision loss
+          if (typeof row[key] === 'string') {
+            result[key] = BigInt(row[key]);
+          }
+          break;
+        case sql.TYPES.Bit:
+          // MSSQL returns BIT as boolean
+          result[key] = row[key] ? 1 : 0;
+          break;
+        // Convert Dates to string
+        case sql.TYPES.Date:
+          result[key] = toISODateString(row[key] as Date);
+          break;
+        case sql.TYPES.Time:
+          result[key] = toISOTimeString(row[key] as Date);
+          break;
+        case sql.TYPES.DateTime:
+        case sql.TYPES.DateTime2:
+        case sql.TYPES.SmallDateTime:
+        case sql.TYPES.DateTimeOffset: // The offset is lost when the driver converts to Date. This needs to be handled in the sql query.
+          const date = row[key] as Date;
+          result[key] = isNaN(date.getTime()) ? null : date.toISOString();
+          break;
+        case sql.TYPES.Binary:
+        case sql.TYPES.VarBinary:
+        case sql.TYPES.Image:
+          result[key] = new Uint8Array(Object.values(row[key]));
+          break;
+        // TODO: Spatial types need to be converted to binary WKB, they are returned as a non standard object currently
+        case sql.TYPES.Geometry:
+        case sql.TYPES.Geography:
+          result[key] = JSON.stringify(row[key]);
+          break;
+        case sql.TYPES.UDT:
+          if (columnMetadata.udt.name === MSSQLUserDefinedType.HIERARCHYID) {
+            result[key] = new Uint8Array(Object.values(row[key]));
+            break;
+          } else {
+            result[key] = row[key];
+          }
+          break;
+        default:
+          result[key] = row[key];
+      }
+    } else {
+      // If the value is null, we just set it to null
+      result[key] = null;
+    }
+  }
+  return toSyncRulesRow(result);
+}
+
+function toISODateString(date: Date): string | null {
+  return isNaN(date.getTime()) ? null : date.toISOString().split('T')[0];
+}
+
+/**
+ *  MSSQL time format is HH:mm:ss[.nnnnnnn]
+ * @param date
+ * @returns
+ */
+function toISOTimeString(date: Date): string | null {
+  return isNaN(date.getTime()) ? null : date.toISOString().split('T')[1].replace('Z', '');
+}
+
+/**
+ * Converts MSSQL type names to SQLite ExpressionType
+ * @param mssqlType - The MSSQL type name (e.g., 'int', 'varchar', 'datetime2')
+ */
+export function toExpressionTypeFromMSSQLType(mssqlType: string | undefined): ExpressionType {
+  if (!mssqlType) {
+    return ExpressionType.TEXT;
+  }
+
+  const baseType = mssqlType.toUpperCase();
+  switch (baseType) {
+    case 'BIT':
+    case 'TINYINT':
+    case 'SMALLINT':
+    case 'INT':
+    case 'INTEGER':
+    case 'BIGINT':
+      return ExpressionType.INTEGER;
+    case 'BINARY':
+    case 'VARBINARY':
+    case 'IMAGE':
+    case 'TIMESTAMP':
+      return ExpressionType.BLOB;
+    case 'FLOAT':
+    case 'REAL':
+    case 'MONEY':
+    case 'SMALLMONEY':
+    case 'DECIMAL':
+    case 'NUMERIC':
+      return ExpressionType.REAL;
+    case 'JSON':
+      return ExpressionType.TEXT;
+    // System and extended types
+    case 'SYSNAME':
+      // SYSNAME is essentially NVARCHAR(128), map to TEXT
+      return ExpressionType.TEXT;
+    case 'HIERARCHYID':
+      // HIERARCHYID is a CLR UDT representing hierarchical data, stored as string representation
+      return ExpressionType.TEXT;
+    case 'GEOMETRY':
+    case 'GEOGRAPHY':
+      // Spatial CLR UDT types, typically stored as WKT (Well-Known Text) strings
+      return ExpressionType.TEXT;
+    case 'VECTOR':
+      // Vector type (SQL Server 2022+), stored as binary data
+      return ExpressionType.BLOB;
+    default:
+      // In addition to the normal text types, includes: VARCHAR, NVARCHAR, CHAR, NCHAR, TEXT, NTEXT, DATE, TIME, DATETIME, DATETIME2, SMALLDATETIME, DATETIMEOFFSET, XML, UNIQUEIDENTIFIER, SQL_VARIANT
+      return ExpressionType.TEXT;
+  }
+}
+
+export interface CDCRowToSqliteRowOptions {
+  row: any;
+  columns: sql.IColumnMetadata;
+}
+// CDC metadata columns in CDCS rows that should be excluded
+const CDC_METADATA_COLUMNS = ['__$operation', '__$start_lsn', '__$end_lsn', '__$seqval', '__$update_mask'];
+/**
+ * Convert CDC row data to SqliteRow format.
+ * CDC rows include table columns plus CDC metadata columns (__$operation, __$start_lsn, etc.)
+ * which we filter out.
+ */
+export function CDCToSqliteRow(options: CDCRowToSqliteRowOptions): SqliteInputRow {
+  const { row, columns } = options;
+  const filteredRow: DatabaseInputRow = {};
+  for (const key in row) {
+    if (!CDC_METADATA_COLUMNS.includes(key)) {
+      filteredRow[key] = row[key];
+    }
+  }
+  return toSqliteInputRow(filteredRow, columns);
+}

package/src/index.ts

@@ -0,0 +1 @@
+export * from './module/MSSQLModule.js';

package/src/module/MSSQLModule.ts

@@ -0,0 +1,82 @@
+import {
+  api,
+  ConfigurationFileSyncRulesProvider,
+  ConnectionTestResult,
+  replication,
+  system,
+  TearDownOptions
+} from '@powersync/service-core';
+import { MSSQLConnectionManagerFactory } from '../replication/MSSQLConnectionManagerFactory.js';
+import * as types from '../types/types.js';
+import { CDCReplicator } from '../replication/CDCReplicator.js';
+import { MSSQLConnectionManager } from '../replication/MSSQLConnectionManager.js';
+import { checkSourceConfiguration } from '../utils/mssql.js';
+import { MSSQLErrorRateLimiter } from '../replication/MSSQLErrorRateLimiter.js';
+import { MSSQLRouteAPIAdapter } from '../api/MSSQLRouteAPIAdapter.js';
+
+export class MSSQLModule extends replication.ReplicationModule<types.MSSQLConnectionConfig> {
+  constructor() {
+    super({
+      name: 'MSSQL',
+      type: types.MSSQL_CONNECTION_TYPE,
+      configSchema: types.MSSQLConnectionConfig
+    });
+  }
+
+  async onInitialized(context: system.ServiceContextContainer): Promise<void> {}
+
+  protected createRouteAPIAdapter(): api.RouteAPI {
+    return new MSSQLRouteAPIAdapter(this.resolveConfig(this.decodedConfig!));
+  }
+
+  protected createReplicator(context: system.ServiceContext): replication.AbstractReplicator {
+    const normalisedConfig = this.resolveConfig(this.decodedConfig!);
+    const syncRuleProvider = new ConfigurationFileSyncRulesProvider(context.configuration.sync_rules);
+    const connectionFactory = new MSSQLConnectionManagerFactory(normalisedConfig);
+
+    return new CDCReplicator({
+      id: this.getDefaultId(normalisedConfig.database),
+      syncRuleProvider: syncRuleProvider,
+      storageEngine: context.storageEngine,
+      metricsEngine: context.metricsEngine,
+      connectionFactory: connectionFactory,
+      rateLimiter: new MSSQLErrorRateLimiter(),
+      pollingOptions: normalisedConfig.cdcPollingOptions
+    });
+  }
+
+  /**
+   * Combines base config with normalized connection settings
+   */
+  private resolveConfig(config: types.MSSQLConnectionConfig): types.ResolvedMSSQLConnectionConfig {
+    return {
+      ...config,
+      ...types.normalizeConnectionConfig(config)
+    };
+  }
+
+  async teardown(options: TearDownOptions): Promise<void> {
+    // No specific teardown required for MSSQL
+  }
+
+  async testConnection(config: types.MSSQLConnectionConfig) {
+    this.decodeConfig(config);
+    const normalizedConfig = this.resolveConfig(this.decodedConfig!);
+    return await MSSQLModule.testConnection(normalizedConfig);
+  }
+
+  static async testConnection(normalizedConfig: types.ResolvedMSSQLConnectionConfig): Promise<ConnectionTestResult> {
+    const connectionManager = new MSSQLConnectionManager(normalizedConfig, { max: 1 });
+    try {
+      const errors = await checkSourceConfiguration(connectionManager);
+      if (errors.length > 0) {
+        throw new Error(errors.join('\n'));
+      }
+    } finally {
+      await connectionManager.end();
+    }
+    return {
+      connectionDescription: normalizedConfig.hostname
+    };
+  }
+}