@powersync/service-module-mssql 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/src/replication/CaptureReconciler.ts

@@ -0,0 +1,186 @@
+import { ErrorCode, ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework';
+import { JsonValue, SourceTableCandidate, storage } from '@powersync/service-core';
+import * as t from 'ts-codec';
+import { toQualifiedTableName } from '../utils/mssql.js';
+import type { MSSQLTableReconciliationContext } from './MSSQLTableReconciliationContext.js';
+import { MSSQLTableReconciliationState } from './MSSQLTableReconciliationContext.js';
+
+/**
+ * The pinned capture instance was dropped and cannot be replaced in this stream.
+ */
+export class CaptureInstanceMissingError extends ServiceError {
+  constructor(message: string) {
+    super(ErrorCode.PSYNC_S1601, message);
+  }
+}
+
+/**
+ * A configured table does not exist or does not have CDC enabled yet.
+ */
+export class SourceTableNotReadyError extends ServiceError {
+  constructor(message: string) {
+    super(ErrorCode.PSYNC_S1602, message);
+  }
+}
+
+/**
+ * A replicated table is unavailable or no longer matches its persisted source binding.
+ */
+export class SourceTableUnavailableError extends ServiceError {
+  constructor(message: string) {
+    super(ErrorCode.PSYNC_S1603, message);
+  }
+}
+
+/**
+ * Persisted capture-table identity. Names are not enough because SQL Server can reuse them.
+ */
+export const MSSQLSourceMetadata = t.object({
+  captureTableObjectId: t.number
+});
+export type MSSQLSourceMetadata = t.Decoded<typeof MSSQLSourceMetadata>;
+
+/**
+ * Parse persisted capture metadata, returning null for legacy records.
+ */
+export function readCaptureMetadata(value: JsonValue): MSSQLSourceMetadata | null {
+  if (value == null) {
+    return null;
+  }
+  return MSSQLSourceMetadata.decode(value as t.Encoded<typeof MSSQLSourceMetadata>);
+}
+
+/**
+ * Reconcile a discovered source-table state against persisted bindings. Ready tables reconcile
+ * against capture instances ordered newest first. New and legacy bindings use the newest instance;
+ * pinned bindings keep their instance or fail if it is gone.
+ */
+export function createCaptureReconciler(context: MSSQLTableReconciliationContext) {
+  return (({ source, candidates }) => {
+    if (context.state === MSSQLTableReconciliationState.TABLE_MISSING) {
+      if (candidates.length > 0) {
+        // The table is missing, but we already recorded it. Display the corresponding error.
+        throw sourceTableUnavailableError(source);
+      }
+      throw new SourceTableNotReadyError(
+        `Source table ${toQualifiedTableName(source.schema, source.name)} from the sync configuration does not ` +
+          `exist. Create the table and enable CDC for it.`
+      );
+    }
+
+    // Apply capture pinning after the shared identity check.
+    const compatible: SourceTableCandidate[] = [];
+    const incompatibleTables: SourceTableCandidate[] = [];
+    for (const candidate of candidates) {
+      if (storage.sourceIdentityCompatible(source, candidate)) {
+        compatible.push(candidate);
+      } else {
+        incompatibleTables.push(candidate);
+      }
+    }
+
+    if (compatible.length === 0) {
+      if (candidates.length > 0) {
+        // An overlapping persisted binding with a different source identity is not a new table.
+        // This can happen when a table is dropped and recreated while replication is stopped.
+        // Reusing the old snapshot or replacing it during a normal job restart would silently
+        // adopt a different physical table without a sync config deployment boundary.
+        throw sourceTableUnavailableError(source);
+      }
+
+      if (context.state === MSSQLTableReconciliationState.CDC_DISABLED) {
+        throw new SourceTableNotReadyError(
+          `CDC is not enabled for source table ${toQualifiedTableName(source.schema, source.name)}, which ` +
+            `matches the sync configuration. Enable CDC for this table to start replicating it.`
+        );
+      }
+
+      // Pin a new binding to the newest instance.
+      const newest = context.captureInstances[0];
+      return {
+        compatibleTables: [],
+        incompatibleTables,
+        newTableValues: { sourceMetadata: captureMetadata(newest.objectId) }
+      };
+    }
+
+    // At least one compatible candidate anchors this process to the discovered source identity.
+    // Any remaining incompatible candidates are stale overlapping records, so storage may remove
+    // them without adopting a replacement table or changing the binding this process uses.
+    if (context.state === MSSQLTableReconciliationState.CDC_DISABLED) {
+      throw new CaptureInstanceMissingError(
+        `CDC is no longer enabled for source table ${toQualifiedTableName(source.schema, source.name)}. ` +
+          `Re-enable CDC, then redeploy the sync config to adopt the replacement capture instance. Its ` +
+          `already-replicated data is retained until the redeployed sync config becomes active.`
+      );
+    }
+
+    const availableInstances = context.captureInstances;
+
+    const pinnedObjectIds = new Set<number>();
+    let metadataFreeCount = 0;
+    for (const candidate of compatible) {
+      const metadata = readCaptureMetadata(candidate.sourceMetadata);
+      if (metadata == null) {
+        metadataFreeCount += 1;
+      } else {
+        pinnedObjectIds.add(metadata.captureTableObjectId);
+      }
+    }
+
+    // Never guess which capture schema existing snapshots belong to.
+    if (pinnedObjectIds.size > 0 && metadataFreeCount > 0) {
+      throw new ReplicationAssertionError(
+        `Source table ${toQualifiedTableName(source.schema, source.name)} has a mixture of legacy (metadata-free) and ` +
+          `capture-instance-pinned records. This is invalid persisted state.`
+      );
+    }
+    // All records for a physical table must use the same capture instance.
+    if (pinnedObjectIds.size > 1) {
+      throw new ReplicationAssertionError(
+        `Source table ${toQualifiedTableName(source.schema, source.name)} has multiple persisted capture identities ` +
+          `(${[...pinnedObjectIds].join(', ')}). This is invalid persisted state.`
+      );
+    }
+
+    if (pinnedObjectIds.size === 0) {
+      // Backfill legacy records with the instance the old code would have selected.
+      const newest = availableInstances[0];
+      const sourceMetadata = captureMetadata(newest.objectId);
+      return {
+        compatibleTables: compatible.map((candidate) => candidate.withSourceMetadata(sourceMetadata)),
+        incompatibleTables,
+        newTableValues: { sourceMetadata }
+      };
+    }
+
+    const [captureTableObjectId] = [...pinnedObjectIds];
+    const available = availableInstances.some((instance) => instance.objectId === captureTableObjectId);
+    if (!available) {
+      // A replacement may capture a different schema, so do not adopt it in place.
+      throw new CaptureInstanceMissingError(
+        `The CDC capture instance (object id ${captureTableObjectId}) for source table ` +
+          `${toQualifiedTableName(source.schema, source.name)} is no longer available. Deploy a new sync config to ` +
+          `replicate this table against a new capture instance.`
+      );
+    }
+    return {
+      compatibleTables: compatible,
+      incompatibleTables,
+      newTableValues: { sourceMetadata: captureMetadata(captureTableObjectId) }
+    };
+  }) satisfies storage.SourceTableCandidateReconciler;
+}
+
+function captureMetadata(captureTableObjectId: number): MSSQLSourceMetadata {
+  return { captureTableObjectId };
+}
+
+function sourceTableUnavailableError(source: { schema: string; name: string }): SourceTableUnavailableError {
+  return new SourceTableUnavailableError(
+    `Table ${toQualifiedTableName(source.schema, source.name)} no longer matches the source table binding ` +
+      `selected by this replication process. It may have been dropped and recreated, renamed, or had its ` +
+      `replication identity changed. Redeploy the sync config to adopt the replacement. Its already-replicated ` +
+      `data is retained until the redeployed sync config becomes active.`
+  );
+}

package/src/replication/MSSQLConnectionManager.ts

@@ -11,6 +11,28 @@ export interface MSSQLConnectionManagerListener {
   onEnded(): void;
 }
 
+export function createConnectionPoolConfig(
+  options: NormalizedMSSQLConnectionConfig,
+  poolOptions: sql.PoolOpts<sql.Connection>
+): sql.config {
+  return {
+    authentication: options.authentication,
+    user: options.username,
+    password: options.password,
+    server: options.hostname,
+    port: options.port,
+    database: options.database,
+    pool: poolOptions,
+    options: {
+      appName: `powersync/${POWERSYNC_VERSION}`,
+      encrypt: true, // Required for Azure
+      trustServerCertificate: options.additionalConfig.trustServerCertificate,
+      ...(options.tls_servername ? { serverName: options.tls_servername } : {}),
+      ...(options.cacert ? { cryptoCredentialsDetails: { ca: options.cacert } } : {})
+    }
+  };
+}
+
 export class MSSQLConnectionManager extends BaseObserver<MSSQLConnectionManagerListener> {
   private readonly pool: sql.ConnectionPool;
 
@@ -20,20 +42,7 @@ export class MSSQLConnectionManager extends BaseObserver<MSSQLConnectionManagerL
   ) {
     super();
     // The pool is lazy - no connections are opened until a query is performed.
-    this.pool = new sql.ConnectionPool({
-      authentication: options.authentication,
-      user: options.username,
-      password: options.password,
-      server: options.hostname,
-      port: options.port,
-      database: options.database,
-      pool: poolOptions,
-      options: {
-        appName: `powersync/${POWERSYNC_VERSION}`,
-        encrypt: true, // Required for Azure
-        trustServerCertificate: options.additionalConfig.trustServerCertificate
-      }
-    });
+    this.pool = new sql.ConnectionPool(createConnectionPoolConfig(options, poolOptions));
   }
 
   public get connectionTag() {

package/src/replication/MSSQLTableReconciliationContext.ts

@@ -0,0 +1,29 @@
+import type { SourceEntityDescriptor } from '@powersync/service-core';
+import type { CaptureInstance } from '../common/CaptureInstance.js';
+
+export enum MSSQLTableReconciliationState {
+  TABLE_MISSING = 'table_missing',
+  CDC_DISABLED = 'cdc_disabled',
+  READY = 'ready'
+}
+
+interface BaseMSSQLTableReconciliationContext {
+  source: SourceEntityDescriptor;
+}
+
+export type MSSQLUnavailableTableReconciliationContext =
+  | (BaseMSSQLTableReconciliationContext & {
+      state: MSSQLTableReconciliationState.TABLE_MISSING;
+    })
+  | (BaseMSSQLTableReconciliationContext & {
+      state: MSSQLTableReconciliationState.CDC_DISABLED;
+    });
+
+export type MSSQLReadyTableReconciliationContext = BaseMSSQLTableReconciliationContext & {
+  state: MSSQLTableReconciliationState.READY;
+  captureInstances: readonly CaptureInstance[];
+};
+
+export type MSSQLTableReconciliationContext =
+  | MSSQLUnavailableTableReconciliationContext
+  | MSSQLReadyTableReconciliationContext;

package/src/replication/SchemaChange.ts

@@ -0,0 +1,79 @@
+import { CaptureInstance } from '../common/CaptureInstance.js';
+import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
+import { SourceTableChangeRef } from '../utils/schema.js';
+
+/**
+ * Schema changes are detected to warn or error on, not applied automatically. Polling cannot be
+ * made atomic with a commit, so the replicated schema stays fixed until the next deploy.
+ *
+ * Drops and renames fail the job because they have no known LSN. Continuing could skip unread
+ * changes. Existing replicated data is kept for the next deploy to clean up.
+ */
+export enum SchemaChangeType {
+  /**
+   * A replicated table was renamed.
+   */
+  TABLE_RENAME = 'table_rename',
+  /**
+   * A replicated table was dropped.
+   */
+  TABLE_DROP = 'table_drop',
+  /**
+   * The source columns changed. The pinned capture schema remains active.
+   */
+  TABLE_COLUMN_CHANGES = 'table_column_changes',
+  /**
+   * A newer capture instance exists, but the pinned one is still available.
+   */
+  NEW_CAPTURE_INSTANCE = 'new_capture_instance',
+  /**
+   * The pinned capture instance is no longer available.
+   */
+  MISSING_CAPTURE_INSTANCE = 'missing_capture_instance'
+}
+
+interface SchemaChangeBase<Type extends SchemaChangeType> {
+  type: Type;
+  /**
+   * The replicated table the change applies to.
+   */
+  table: MSSQLSourceTable;
+}
+
+export interface TableRenameChange extends SchemaChangeBase<SchemaChangeType.TABLE_RENAME> {
+  /**
+   * The name the table was renamed to. Only used in the error reported for the rename.
+   */
+  newTable: SourceTableChangeRef;
+}
+
+export type TableDropChange = SchemaChangeBase<SchemaChangeType.TABLE_DROP>;
+
+export interface TableColumnChanges extends SchemaChangeBase<SchemaChangeType.TABLE_COLUMN_CHANGES> {
+  /**
+   * The pinned capture instance the drift was observed against.
+   */
+  captureInstance: CaptureInstance;
+}
+
+export interface NewCaptureInstanceChange extends SchemaChangeBase<SchemaChangeType.NEW_CAPTURE_INSTANCE> {
+  /**
+   * The newer instance that is available but will not be adopted by this stream.
+   */
+  newCaptureInstance: CaptureInstance;
+}
+
+export interface MissingCaptureInstanceChange extends SchemaChangeBase<SchemaChangeType.MISSING_CAPTURE_INSTANCE> {
+  /**
+   * A newer instance that replaced the pinned one, if CDC is still enabled for the table. Only used
+   * to report the right recovery step.
+   */
+  replacementInstance?: CaptureInstance;
+}
+
+export type SchemaChange =
+  | TableRenameChange
+  | TableDropChange
+  | TableColumnChanges
+  | NewCaptureInstanceChange
+  | MissingCaptureInstanceChange;

package/src/types/types.ts

@@ -8,6 +8,10 @@ export const DEFAULT_POLLING_BATCH_SIZE = 10;
 export const DEFAULT_POLLING_INTERVAL_MS = 1000;
 export const MSSQL_CONNECTION_TYPE = 'mssql' as const;
 
+const DEFAULT_HEARTBEAT_INTERVAL_SECONDS = 60;
+const MIN_HEARTBEAT_INTERVAL_SECONDS = 5;
+const MAX_HEARTBEAT_INTERVAL_SECONDS = 60;
+
 export const AzureActiveDirectoryServicePrincipalSecret = t.object({
   type: t.literal('azure-active-directory-service-principal-secret'),
   options: t.object({
@@ -89,11 +93,19 @@ export interface NormalizedMSSQLConnectionConfig {
   database: string;
   schema?: string;
 
+  /** PEM-encoded certificate authority used to validate the SQL Server certificate. */
+  cacert?: string;
+
+  /** Server name to use when validating the SQL Server certificate. */
+  tls_servername?: string;
+
   authentication?: Authentication;
 
   lookup?: LookupFunction;
 
   additionalConfig: AdditionalConfig;
+
+  heartbeat_interval_seconds: number;
 }
 
 export const MSSQLConnectionConfig = service_types.configFile.DataSourceConfig.and(
@@ -107,10 +119,20 @@ export const MSSQLConnectionConfig = service_types.configFile.DataSourceConfig.a
     hostname: t.string.optional(),
     port: service_types.configFile.portCodec.optional(),
 
+    /** PEM-encoded certificate authority used to validate the SQL Server certificate. */
+    cacert: t.string.optional(),
+
+    /** Server name to use when validating the SQL Server certificate. */
+    tls_servername: t.string.optional(),
+
     authentication: Authentication.optional(),
 
     reject_ip_ranges: t.array(t.string).optional(),
-    additionalConfig: AdditionalConfig.optional()
+    additionalConfig: AdditionalConfig.optional(),
+    /**
+     * Interval in seconds between source connection heartbeats. Null or omitted uses the default.
+     */
+    heartbeat_interval_seconds: service_types.orNull(t.number).optional()
   })
 );
 
@@ -169,6 +191,13 @@ export function normalizeConnectionConfig(options: MSSQLConnectionConfig): Norma
     throw new ServiceError(ErrorCode.PSYNC_S1105, `MSSQL connection: database required`);
   }
 
+  if (options.cacert && options.additionalConfig?.trustServerCertificate) {
+    throw new ServiceError(
+      ErrorCode.PSYNC_S1604,
+      'MSSQL connection: cacert cannot be used with trustServerCertificate because certificate validation would be disabled'
+    );
+  }
+
   const lookup = makeHostnameLookupFunction(hostname, { reject_ip_ranges: options.reject_ip_ranges ?? [] });
 
   return {
@@ -180,6 +209,8 @@ export function normalizeConnectionConfig(options: MSSQLConnectionConfig): Norma
     hostname,
     port,
     database,
+    cacert: options.cacert,
+    tls_servername: options.tls_servername,
 
     lookup,
     authentication: options.authentication,
@@ -188,10 +219,24 @@ export function normalizeConnectionConfig(options: MSSQLConnectionConfig): Norma
       pollingIntervalMs: options.additionalConfig?.pollingIntervalMs ?? DEFAULT_POLLING_INTERVAL_MS,
       pollingBatchSize: options.additionalConfig?.pollingBatchSize ?? DEFAULT_POLLING_BATCH_SIZE,
       trustServerCertificate: options.additionalConfig?.trustServerCertificate ?? false
-    }
+    },
+    heartbeat_interval_seconds: normalizeHeartbeatInterval(options.heartbeat_interval_seconds)
   } satisfies NormalizedMSSQLConnectionConfig;
 }
 
+function normalizeHeartbeatInterval(value: number | null | undefined): number {
+  if (value == null) {
+    return DEFAULT_HEARTBEAT_INTERVAL_SECONDS;
+  }
+  if (!Number.isFinite(value) || value < MIN_HEARTBEAT_INTERVAL_SECONDS || value > MAX_HEARTBEAT_INTERVAL_SECONDS) {
+    throw new ServiceError(
+      ErrorCode.PSYNC_S1109,
+      `MSSQL connection: heartbeat_interval_seconds must be between ${MIN_HEARTBEAT_INTERVAL_SECONDS} and ${MAX_HEARTBEAT_INTERVAL_SECONDS} seconds`
+    );
+  }
+  return value;
+}
+
 export function baseUri(config: ResolvedMSSQLConnectionConfig) {
   return `mssql://${config.hostname}:${config.port}/${config.database}`;
 }

package/src/utils/mssql.ts

@@ -351,7 +351,7 @@ export async function getDebugTableInfo(options: GetDebugTableInfoOptions): Prom
     if (captureInstanceDetails && captureInstanceDetails.instances[0].pendingSchemaChanges.length > 0) {
       schemaDriftError = {
         level: 'warning',
-        message: `Source table ${toQualifiedTableName(schema, table.name)} has schema changes not reflected in the CDC capture instance. Please disable and re-enable CDC on the source table to update the capture instance schema.
+        message: `Source table ${toQualifiedTableName(schema, table.name)} has schema changes not reflected in the CDC capture instance. Create a replacement CDC capture instance with the intended columns, update the sync config if needed, and deploy a new sync config. If the change requires disabling CDC, re-enable CDC before deploying.
         Pending schema changes: ${captureInstanceDetails.instances[0].pendingSchemaChanges.join(', \n')}`
       };
     }

package/src/utils/schema.ts

@@ -1,4 +1,4 @@
-import { logger } from '@powersync/lib-services-framework';
+import { ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
 import { TablePattern } from '@powersync/service-sync-rules';
 import sql from 'mssql';
 import { MSSQLConnectionManager } from '../replication/MSSQLConnectionManager.js';
@@ -139,70 +139,76 @@ export async function getReplicationIdentityColumns(
 }
 
 export interface SourceTableChangeRef {
-  objectId: number | string | undefined;
+  /**
+   * `sys.tables.object_id`, which SQL Server always reports as an int. Narrower than
+   * `SourceEntityDescriptor.objectId`, which has to accommodate other source databases.
+   */
+  objectId: number;
   schema: string;
   name: string;
 }
 
 export type ResolvedTable = SourceTableChangeRef;
 
+export const SCHEMA_WILDCARD_MESSAGE =
+  'Schema wildcards ("%") in table patterns are not supported for SQL Server connections.';
+
+/**
+ * Wildcards would allow the replicated table set to change between polls, so they are rejected.
+ */
+export function tableWildcardMessage(tablePattern: TablePattern): string {
+  return (
+    `Table wildcards ("%") are not supported for SQL Server connections: "${tablePattern.tablePattern}". ` +
+    `List each table explicitly.`
+  );
+}
+
+/**
+ * Returns the reason this pattern cannot be replicated, or null if it is supported.
+ */
+export function unsupportedTablePatternMessage(tablePattern: TablePattern): string | null {
+  if (tablePattern.isSchemaWildcard) {
+    return SCHEMA_WILDCARD_MESSAGE;
+  }
+  if (tablePattern.isWildcard) {
+    return tableWildcardMessage(tablePattern);
+  }
+  return null;
+}
+
 export async function getTablesFromPattern(
   connectionManager: MSSQLConnectionManager,
   tablePattern: TablePattern
 ): Promise<ResolvedTable[]> {
-  if (tablePattern.isWildcard) {
-    const { recordset: tableResults } = await connectionManager.query(
-      `
-        SELECT
-          tbl.name      AS [table],
-          sch.name      AS [schema],
-          tbl.object_id AS object_id
-        FROM sys.tables tbl
-          JOIN sys.schemas sch ON tbl.schema_id = sch.schema_id
-        WHERE sch.name = @schema
-          AND tbl.name LIKE @tablePattern
-      `,
-      [
-        { name: 'schema', type: sql.VarChar(sql.MAX), value: tablePattern.schema },
-        { name: 'tablePattern', type: sql.VarChar(sql.MAX), value: tablePattern.tablePattern }
-      ]
-    );
-
-    return tableResults
-      .map((row) => {
-        return {
-          objectId: row.object_id,
-          schema: row.schema,
-          name: row.table
-        };
-      })
-      .filter((table: ResolvedTable) => table.name.startsWith(tablePattern.tablePrefix));
-  } else {
-    const { recordset: tableResults } = await connectionManager.query(
-      `
-        SELECT
-          tbl.name      AS [table],
-          sch.name      AS [schema],
-          tbl.object_id AS object_id
-        FROM sys.tables tbl
-          JOIN sys.schemas sch ON tbl.schema_id = sch.schema_id
-          WHERE sch.name = @schema
-          AND tbl.name = @tablePattern
-      `,
-      [
-        { name: 'schema', type: sql.VarChar(sql.MAX), value: tablePattern.schema },
-        { name: 'tablePattern', type: sql.VarChar(sql.MAX), value: tablePattern.tablePattern }
-      ]
-    );
-
-    return tableResults.map((row) => {
-      return {
-        objectId: row.object_id,
-        schema: row.schema,
-        name: row.table
-      };
-    });
+  const unsupported = unsupportedTablePatternMessage(tablePattern);
+  if (unsupported != null) {
+    throw new ServiceError(ErrorCode.PSYNC_R2201, unsupported);
   }
+
+  const { recordset: tableResults } = await connectionManager.query(
+    `
+      SELECT
+        tbl.name      AS [table],
+        sch.name      AS [schema],
+        tbl.object_id AS object_id
+      FROM sys.tables tbl
+        JOIN sys.schemas sch ON tbl.schema_id = sch.schema_id
+        WHERE sch.name = @schema
+        AND tbl.name = @tablePattern
+    `,
+    [
+      { name: 'schema', type: sql.VarChar(sql.MAX), value: tablePattern.schema },
+      { name: 'tablePattern', type: sql.VarChar(sql.MAX), value: tablePattern.tablePattern }
+    ]
+  );
+
+  return tableResults.map((row) => {
+    return {
+      objectId: row.object_id,
+      schema: row.schema,
+      name: row.table
+    };
+  });
 }
 
 export interface GetPendingSchemaChangesOptions {