@powersync/service-module-mssql 0.0.1

package/src/replication/MSSQLConnectionManager.ts

@@ -0,0 +1,113 @@
+import { BaseObserver, logger } from '@powersync/lib-services-framework';
+import sql from 'mssql';
+import { NormalizedMSSQLConnectionConfig } from '../types/types.js';
+import { POWERSYNC_VERSION } from '@powersync/service-core';
+import { MSSQLParameter } from '../types/mssql-data-types.js';
+import { addParameters } from '../utils/mssql.js';
+
+export const DEFAULT_SCHEMA = 'dbo';
+
+export interface MSSQLConnectionManagerListener {
+  onEnded(): void;
+}
+
+export class MSSQLConnectionManager extends BaseObserver<MSSQLConnectionManagerListener> {
+  private readonly pool: sql.ConnectionPool;
+
+  constructor(
+    public options: NormalizedMSSQLConnectionConfig,
+    poolOptions: sql.PoolOpts<sql.Connection>
+  ) {
+    super();
+    // The pool is lazy - no connections are opened until a query is performed.
+    this.pool = new sql.ConnectionPool({
+      authentication: options.authentication,
+      user: options.username,
+      password: options.password,
+      server: options.hostname,
+      port: options.port,
+      database: options.database,
+      pool: poolOptions,
+      options: {
+        appName: `powersync/${POWERSYNC_VERSION}`,
+        encrypt: true, // Required for Azure
+        trustServerCertificate: options.trustServerCertificate
+      }
+    });
+  }
+
+  public get connectionTag() {
+    return this.options.tag;
+  }
+
+  public get connectionId() {
+    return this.options.id;
+  }
+
+  public get databaseName() {
+    return this.options.database;
+  }
+
+  public get schema() {
+    return this.options.schema ?? DEFAULT_SCHEMA;
+  }
+
+  private async ensureConnected(): Promise<void> {
+    await this.pool.connect();
+  }
+
+  async createTransaction(): Promise<sql.Transaction> {
+    await this.ensureConnected();
+    return this.pool.transaction();
+  }
+
+  async createRequest(): Promise<sql.Request> {
+    await this.ensureConnected();
+    return this.pool.request();
+  }
+
+  async query(query: string, parameters?: MSSQLParameter[]): Promise<sql.IResult<any>> {
+    await this.ensureConnected();
+    for (let tries = 2; ; tries--) {
+      try {
+        logger.debug(`Executing query: ${query}`);
+        let request = this.pool.request();
+        if (parameters) {
+          request = addParameters(request, parameters);
+        }
+        return await request.query(query);
+      } catch (e) {
+        if (tries == 1) {
+          throw e;
+        }
+        logger.warn('Query error, retrying..', e);
+      }
+    }
+  }
+
+  async execute(procedure: string, parameters?: MSSQLParameter[]): Promise<sql.IProcedureResult<any>> {
+    await this.ensureConnected();
+    let request = this.pool.request();
+    if (parameters) {
+      if (parameters) {
+        request = addParameters(request, parameters);
+      }
+    }
+    return request.execute(procedure);
+  }
+
+  async end(): Promise<void> {
+    if (this.pool.connected) {
+      try {
+        await this.pool.close();
+      } catch (error) {
+        // We don't particularly care if any errors are thrown when shutting down the pool
+        logger.warn('Error shutting down MSSQL connection pool', error);
+      } finally {
+        this.iterateListeners((listener) => {
+          listener.onEnded?.();
+        });
+      }
+    }
+  }
+}

package/src/replication/MSSQLConnectionManagerFactory.ts

@@ -0,0 +1,33 @@
+import { logger } from '@powersync/lib-services-framework';
+import { ResolvedMSSQLConnectionConfig } from '../types/types.js';
+import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
+import sql from 'mssql';
+
+export class MSSQLConnectionManagerFactory {
+  private readonly connectionManagers: Set<MSSQLConnectionManager>;
+  public readonly connectionConfig: ResolvedMSSQLConnectionConfig;
+
+  constructor(connectionConfig: ResolvedMSSQLConnectionConfig) {
+    this.connectionConfig = connectionConfig;
+    this.connectionManagers = new Set<MSSQLConnectionManager>();
+  }
+
+  create(poolOptions: sql.PoolOpts<sql.Connection>) {
+    const manager = new MSSQLConnectionManager(this.connectionConfig, poolOptions);
+    manager.registerListener({
+      onEnded: () => {
+        this.connectionManagers.delete(manager);
+      }
+    });
+    this.connectionManagers.add(manager);
+    return manager;
+  }
+
+  async shutdown() {
+    logger.info('Shutting down MSSQL connection Managers...');
+    for (const manager of this.connectionManagers.values()) {
+      await manager.end();
+    }
+    logger.info('MSSQL connection Managers shutdown completed.');
+  }
+}

package/src/replication/MSSQLErrorRateLimiter.ts

@@ -0,0 +1,36 @@
+import { ErrorRateLimiter } from '@powersync/service-core';
+import { setTimeout } from 'timers/promises';
+
+export class MSSQLErrorRateLimiter implements ErrorRateLimiter {
+  nextAllowed: number = Date.now();
+
+  async waitUntilAllowed(options?: { signal?: AbortSignal | undefined } | undefined): Promise<void> {
+    const delay = Math.max(0, this.nextAllowed - Date.now());
+    // Minimum delay between connections, even without errors
+    this.setDelay(500);
+    await setTimeout(delay, undefined, { signal: options?.signal });
+  }
+
+  mayPing(): boolean {
+    return Date.now() >= this.nextAllowed;
+  }
+
+  reportError(e: any): void {
+    const message = (e.message as string) ?? '';
+    if (message.includes('password authentication failed')) {
+      this.setDelay(900_000);
+    } else if (message.includes('ENOTFOUND')) {
+      // DNS lookup issue - incorrect URI or deleted instance
+      this.setDelay(120_000);
+    } else if (message.includes('ECONNREFUSED')) {
+      // Could be fail2ban or similar
+      this.setDelay(120_000);
+    } else {
+      this.setDelay(30_000);
+    }
+  }
+
+  private setDelay(delay: number) {
+    this.nextAllowed = Math.max(this.nextAllowed, Date.now() + delay);
+  }
+}

package/src/replication/MSSQLSnapshotQuery.ts

@@ -0,0 +1,230 @@
+import { bson, ColumnDescriptor, SourceTable } from '@powersync/service-core';
+import { SqliteValue } from '@powersync/service-sync-rules';
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { MSSQLBaseType } from '../types/mssql-data-types.js';
+import sql from 'mssql';
+import { escapeIdentifier } from '../utils/mssql.js';
+import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
+
+export interface MSSQLSnapshotQuery {
+  initialize(): Promise<void>;
+
+  /**
+   *  Returns an async iterable iterator that yields the column metadata for the query followed by rows of data.
+   */
+  next(): AsyncIterableIterator<sql.IColumnMetadata | sql.IRecordSet<any>>;
+}
+
+export type PrimaryKeyValue = Record<string, SqliteValue>;
+
+/**
+ * Snapshot query using a plain SELECT * FROM table
+ *
+ * This supports all tables but does not efficiently resume the snapshot
+ * if the process is restarted.
+ */
+export class SimpleSnapshotQuery implements MSSQLSnapshotQuery {
+  public constructor(
+    private readonly transaction: sql.Transaction,
+    private readonly table: MSSQLSourceTable
+  ) {}
+
+  public async initialize(): Promise<void> {}
+
+  public async *next(): AsyncIterableIterator<sql.IColumnMetadata | sql.IRecordSet<any>> {
+    const metadataRequest = this.transaction.request();
+    metadataRequest.stream = true;
+    const metadataPromise = new Promise<sql.IColumnMetadata>((resolve, reject) => {
+      metadataRequest.on('recordset', resolve);
+      metadataRequest.on('error', reject);
+    });
+
+    metadataRequest.query(`SELECT TOP(0) * FROM ${this.table.toQualifiedName()}`);
+
+    const columnMetadata: sql.IColumnMetadata = await metadataPromise;
+    yield columnMetadata;
+
+    const request = this.transaction.request();
+    const stream = request.toReadableStream();
+
+    request.query(`SELECT * FROM ${this.table.toQualifiedName()}`);
+
+    // MSSQL only streams one row at a time
+    for await (const row of stream) {
+      yield row;
+    }
+  }
+}
+
+/**
+ * Performs a table snapshot query, batching by ranges of primary key data.
+ *
+ * This may miss some rows if they are modified during the snapshot query.
+ * In that case, replication will pick up those rows afterward,
+ * possibly resulting in an IdSnapshotQuery.
+ *
+ * Currently, this only supports a table with a single primary key column,
+ * of a select few types.
+ */
+export class BatchedSnapshotQuery implements MSSQLSnapshotQuery {
+  /**
+   * Primary key types that we support for batched snapshots.
+   *
+   * Can expand this over time as we add more tests,
+   * and ensure there are no issues with type conversion.
+   */
+  static SUPPORTED_TYPES = [
+    MSSQLBaseType.TEXT,
+    MSSQLBaseType.NTEXT,
+    MSSQLBaseType.VARCHAR,
+    MSSQLBaseType.NVARCHAR,
+    MSSQLBaseType.CHAR,
+    MSSQLBaseType.NCHAR,
+    MSSQLBaseType.UNIQUEIDENTIFIER,
+    MSSQLBaseType.TINYINT,
+    MSSQLBaseType.SMALLINT,
+    MSSQLBaseType.INT,
+    MSSQLBaseType.BIGINT
+  ];
+
+  static supports(table: SourceTable | MSSQLSourceTable): boolean {
+    const sourceTable = table instanceof MSSQLSourceTable ? table.sourceTable : table;
+    if (sourceTable.replicaIdColumns.length != 1) {
+      return false;
+    }
+    const primaryKey = sourceTable.replicaIdColumns[0];
+
+    return primaryKey.typeId != null && BatchedSnapshotQuery.SUPPORTED_TYPES.includes(Number(primaryKey.typeId));
+  }
+
+  private readonly key: ColumnDescriptor;
+  lastKey: string | bigint | null = null;
+
+  public constructor(
+    private readonly transaction: sql.Transaction,
+    private readonly table: MSSQLSourceTable,
+    private readonly batchSize: number = 10_000,
+    lastKeySerialized: Uint8Array | null
+  ) {
+    this.key = table.sourceTable.replicaIdColumns[0];
+
+    if (lastKeySerialized != null) {
+      this.lastKey = this.deserializeKey(lastKeySerialized);
+    }
+  }
+
+  public async initialize(): Promise<void> {
+    // No-op
+  }
+
+  public getLastKeySerialized(): Uint8Array {
+    return bson.serialize({ [this.key.name]: this.lastKey });
+  }
+
+  public async *next(): AsyncIterableIterator<sql.IColumnMetadata | sql.IRecordSet<any>> {
+    const escapedKeyName = escapeIdentifier(this.key.name);
+    const metadataRequest = this.transaction.request();
+    metadataRequest.stream = true;
+    const metadataPromise = new Promise<sql.IColumnMetadata>((resolve, reject) => {
+      metadataRequest.on('recordset', resolve);
+      metadataRequest.on('error', reject);
+    });
+    metadataRequest.query(`SELECT TOP(0) * FROM ${this.table.toQualifiedName()}`);
+
+    const columnMetadata: sql.IColumnMetadata = await metadataPromise;
+
+    const foundPrimaryKey = columnMetadata[this.key.name];
+    if (!foundPrimaryKey) {
+      throw new Error(
+        `Cannot find primary key column ${this.key.name} in results. Keys: ${Object.keys(columnMetadata.columns).join(', ')}`
+      );
+    }
+
+    yield columnMetadata;
+
+    const request = this.transaction.request();
+    const stream = request.toReadableStream();
+    if (this.lastKey == null) {
+      request.query(`SELECT TOP(${this.batchSize}) * FROM ${this.table.toQualifiedName()} ORDER BY ${escapedKeyName}`);
+    } else {
+      if (this.key.typeId == null) {
+        throw new Error(`typeId required for primary key ${this.key.name}`);
+      }
+      request
+        .input('lastKey', this.lastKey)
+        .query(
+          `SELECT TOP(${this.batchSize}) * FROM ${this.table.toQualifiedName()} WHERE ${escapedKeyName} > @lastKey ORDER BY ${escapedKeyName}`
+        );
+    }
+
+    // MSSQL only streams one row at a time
+    for await (const row of stream) {
+      this.lastKey = row[this.key.name];
+      yield row;
+    }
+  }
+
+  private deserializeKey(key: Uint8Array) {
+    const decoded = bson.deserialize(key, { useBigInt64: true });
+    const keys = Object.keys(decoded);
+    if (keys.length != 1) {
+      throw new ServiceAssertionError(`Multiple keys found: ${keys.join(', ')}`);
+    }
+    if (keys[0] != this.key.name) {
+      throw new ServiceAssertionError(`Key name mismatch: expected ${this.key.name}, got ${keys[0]}`);
+    }
+
+    return decoded[this.key.name];
+  }
+}
+
+/**
+ * This performs a snapshot query using a list of primary keys.
+ *
+ * This is not used for general snapshots, but is used when we need to re-fetch specific rows
+ * during streaming replication.
+ */
+export class IdSnapshotQuery implements MSSQLSnapshotQuery {
+  static supports(table: SourceTable | MSSQLSourceTable) {
+    // We have the same requirements as BatchedSnapshotQuery.
+    // This is typically only used as a fallback when ChunkedSnapshotQuery
+    // skipped some rows.
+    return BatchedSnapshotQuery.supports(table);
+  }
+
+  public constructor(
+    private readonly transaction: sql.Transaction,
+    private readonly table: MSSQLSourceTable,
+    private readonly keys: PrimaryKeyValue[]
+  ) {}
+
+  public async initialize(): Promise<void> {
+    // No-op
+  }
+
+  public async *next(): AsyncIterableIterator<sql.IColumnMetadata | sql.IRecordSet<any>> {
+    const metadataRequest = this.transaction.request();
+    metadataRequest.stream = true;
+    const metadataPromise = new Promise<sql.IColumnMetadata>((resolve, reject) => {
+      metadataRequest.on('recordset', resolve);
+      metadataRequest.on('error', reject);
+    });
+    metadataRequest.query(`SELECT TOP(0) * FROM ${this.table.toQualifiedName()}`);
+    const columnMetadata: sql.IColumnMetadata = await metadataPromise;
+    yield columnMetadata;
+
+    const keyDefinition = this.table.sourceTable.replicaIdColumns[0];
+    const ids = this.keys.map((record) => record[keyDefinition.name]);
+
+    const request = this.transaction.request();
+    const stream = request.toReadableStream();
+    request
+      .input('ids', ids)
+      .query(`SELECT * FROM ${this.table.toQualifiedName()} WHERE ${escapeIdentifier(keyDefinition.name)} = @ids`);
+
+    // MSSQL only streams one row at a time
+    for await (const row of stream) {
+      yield row;
+    }
+  }
+}

package/src/types/mssql-data-types.ts

@@ -0,0 +1,79 @@
+import { ColumnDescriptor } from '@powersync/service-core';
+import { ISqlType } from 'mssql';
+
+export interface MSSQLColumnDescriptor extends ColumnDescriptor {
+  /** The underlying system type id. For base types system type id == user type id */
+  typeId: number;
+  /** The unique user type id that uniquely identifies the type  */
+  userTypeId: number;
+  // /** The name of the user/alias type. For example SYSNAME, GEOMETRY */
+  // userTypeName: string;
+}
+
+/** The shared system type id for all CLR_UDT types in SQL Server */
+export const CLR_UDT_TYPE_ID = 240;
+
+/**
+ *  Enum mapping the base MSSQL data types to their system type IDs.
+ */
+export enum MSSQLBaseType {
+  IMAGE = 34,
+  TEXT = 35,
+  UNIQUEIDENTIFIER = 36,
+  DATE = 40,
+  TIME = 41,
+  DATETIME2 = 42,
+  DATETIMEOFFSET = 43,
+  TINYINT = 48,
+  SMALLINT = 52,
+  INT = 56,
+  SMALLDATETIME = 58,
+  REAL = 59,
+  MONEY = 60,
+  DATETIME = 61,
+  FLOAT = 62,
+  SQL_VARIANT = 98,
+  NTEXT = 99,
+  BIT = 104,
+  DECIMAL = 106,
+  NUMERIC = 108,
+  SMALLMONEY = 122,
+  BIGINT = 127,
+  VARBINARY = 165,
+  VARCHAR = 167,
+  BINARY = 173,
+  CHAR = 175,
+  TIMESTAMP = 189,
+  NVARCHAR = 231,
+  NCHAR = 239,
+  XML = 241,
+  JSON = 244
+}
+
+/**
+ *  Enum mapping some of the extended user-defined MSSQL data types to their user type IDs.
+ */
+export enum MSSQLExtendedUserType {
+  // VARBINARY system type [155]
+  VECTOR = 255,
+  // NVARCHAR system type [231]
+  SYSNAME = 256,
+  // CLR_UDT system type [240]
+  HIERARCHYID = 128,
+  // CLR_UDT system type [240]
+  GEOMETRY = 129,
+  // CLR_UDT system type [240]
+  GEOGRAPHY = 130
+}
+
+export enum MSSQLUserDefinedType {
+  VECTOR = 'vector',
+  SYSNAME = 'sysname',
+  HIERARCHYID = 'hierarchyid'
+}
+
+export interface MSSQLParameter {
+  name: string;
+  value: any;
+  type?: (() => ISqlType) | ISqlType;
+}