@powersync/service-module-mssql 0.6.1 → 0.6.3

package/src/replication/CDCStream.ts

@@ -1,20 +1,30 @@
 import {
   container,
   DatabaseConnectionError,
+  logger as defaultLogger,
   ErrorCode,
   Logger,
-  logger as defaultLogger,
   ReplicationAbortedError,
   ReplicationAssertionError
 } from '@powersync/lib-services-framework';
-import { getUuidReplicaIdentityBson, MetricsEngine, SourceEntityDescriptor, storage } from '@powersync/service-core';
+import {
+  getUuidReplicaIdentityBson,
+  MetricsEngine,
+  ReplicationLagTracker,
+  SourceEntityDescriptor,
+  storage
+} from '@powersync/service-core';
 
 import { HydratedSyncRules, SqliteInputRow, SqliteRow, TablePattern } from '@powersync/service-sync-rules';
 
 import { ReplicationMetric } from '@powersync/service-types';
-import { BatchedSnapshotQuery, MSSQLSnapshotQuery, SimpleSnapshotQuery } from './MSSQLSnapshotQuery.js';
-import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
-import { getReplicationIdentityColumns, getTablesFromPattern, ResolvedTable } from '../utils/schema.js';
+import sql from 'mssql';
+import { CaptureInstance } from '../common/CaptureInstance.js';
+import { LSN } from '../common/LSN.js';
+import { CDCToSqliteRow, toSqliteInputRow } from '../common/mssqls-to-sqlite.js';
+import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
+import { MSSQLSourceTableCache } from '../common/MSSQLSourceTableCache.js';
+import { AdditionalConfig } from '../types/types.js';
 import {
   checkRetentionThresholds,
   checkSourceConfiguration,
@@ -24,14 +34,10 @@ import {
   getLatestReplicatedLSN,
   isIColumnMetadata
 } from '../utils/mssql.js';
-import sql from 'mssql';
-import { CDCToSqliteRow, toSqliteInputRow } from '../common/mssqls-to-sqlite.js';
-import { LSN } from '../common/LSN.js';
-import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
-import { MSSQLSourceTableCache } from '../common/MSSQLSourceTableCache.js';
+import { getReplicationIdentityColumns, getTablesFromPattern, ResolvedTable } from '../utils/schema.js';
 import { CDCEventHandler, CDCPoller, SchemaChange, SchemaChangeType } from './CDCPoller.js';
-import { AdditionalConfig } from '../types/types.js';
-import { CaptureInstance } from '../common/CaptureInstance.js';
+import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
+import { BatchedSnapshotQuery, MSSQLSnapshotQuery, SimpleSnapshotQuery } from './MSSQLSnapshotQuery.js';
 
 export interface CDCStreamOptions {
   connections: MSSQLConnectionManager;
@@ -101,16 +107,7 @@ export class CDCStream {
 
   public tableCache = new MSSQLSourceTableCache();
 
-  /**
-   * Time of the oldest uncommitted change, according to the source db.
-   * This is used to determine the replication lag.
-   */
-  private oldestUncommittedChange: Date | null = null;
-  /**
-   * Keep track of whether we have done a commit or keepalive yet.
-   * We can only compute replication lag if isStartingReplication == false, or oldestUncommittedChange is present.
-   */
-  public isStartingReplication = true;
+  private replicationLag = new ReplicationLagTracker();
 
   constructor(private options: CDCStreamOptions) {
     this.logger = options.logger ?? defaultLogger;
@@ -128,6 +125,10 @@ export class CDCStream {
     return this.abortSignal.aborted;
   }
 
+  get isStartingReplication() {
+    return this.replicationLag.isStartingReplication;
+  }
+
   get defaultSchema() {
     return this.connections.schema;
   }
@@ -672,10 +673,12 @@ export class CDCStream {
         this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1);
       },
       onCommit: async (lsn: string, transactionCount: number) => {
-        const { checkpointBlocked } = await batch.commit(lsn);
+        const { checkpointBlocked } = await batch.commit(lsn, {
+          oldestUncommittedChange: this.replicationLag.oldestUncommittedChange
+        });
         this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(transactionCount);
         if (!checkpointBlocked) {
-          this.isStartingReplication = false;
+          this.replicationLag.markCommitted();
         }
       },
       onSchemaChange: async (schemaChange: SchemaChange) => {
@@ -800,17 +803,8 @@ export class CDCStream {
     return this.syncRules.applyRowContext<never>(inputRow);
   }
 
-  async getReplicationLagMillis(): Promise<number | undefined> {
-    if (this.oldestUncommittedChange == null) {
-      if (this.isStartingReplication) {
-        // We don't have anything to compute replication lag with yet.
-        return undefined;
-      } else {
-        // We don't have any uncommitted changes, so replication is up-to-date.
-        return 0;
-      }
-    }
-    return Date.now() - this.oldestUncommittedChange.getTime();
+  getReplicationLagMillis(): number | undefined {
+    return this.replicationLag.getLagMillis();
   }
 
   private touch() {
@@ -823,7 +817,7 @@ export class CDCStream {
    *  Creates an update in the source database to ensure regular checkpoints via the CDC
    */
   public async keepAlive() {
-    if (!this.isStartingReplication && !this.stopped) {
+    if (!this.replicationLag.isStartingReplication && !this.stopped) {
       await createCheckpoint(this.connections);
     }
   }

package/src/replication/MSSQLConnectionManager.ts

@@ -1,8 +1,8 @@
 import { BaseObserver, logger } from '@powersync/lib-services-framework';
-import sql from 'mssql';
-import { NormalizedMSSQLConnectionConfig } from '../types/types.js';
 import { POWERSYNC_VERSION } from '@powersync/service-core';
+import sql from 'mssql';
 import { MSSQLParameter } from '../types/mssql-data-types.js';
+import { NormalizedMSSQLConnectionConfig } from '../types/types.js';
 import { addParameters } from '../utils/mssql.js';
 
 export const DEFAULT_SCHEMA = 'dbo';

package/src/replication/MSSQLConnectionManagerFactory.ts

@@ -1,7 +1,7 @@
 import { logger } from '@powersync/lib-services-framework';
+import sql from 'mssql';
 import { ResolvedMSSQLConnectionConfig } from '../types/types.js';
 import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
-import sql from 'mssql';
 
 export class MSSQLConnectionManagerFactory {
   private readonly connectionManagers: Set<MSSQLConnectionManager>;

package/src/replication/MSSQLSnapshotQuery.ts

@@ -1,9 +1,9 @@
-import { bson, ColumnDescriptor, SourceTable } from '@powersync/service-core';
 import { ServiceAssertionError } from '@powersync/lib-services-framework';
-import { MSSQLBaseType } from '../types/mssql-data-types.js';
+import { bson, ColumnDescriptor, SourceTable } from '@powersync/service-core';
 import sql from 'mssql';
-import { escapeIdentifier } from '../utils/mssql.js';
 import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
+import { MSSQLBaseType } from '../types/mssql-data-types.js';
+import { escapeIdentifier } from '../utils/mssql.js';
 
 export interface MSSQLSnapshotQuery {
   initialize(): Promise<void>;

package/src/utils/mssql.ts

@@ -1,21 +1,21 @@
+import { logger } from '@powersync/lib-services-framework';
+import * as sync_rules from '@powersync/service-sync-rules';
+import { SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
+import * as service_types from '@powersync/service-types';
 import sql from 'mssql';
 import { coerce, gte } from 'semver';
-import { logger } from '@powersync/lib-services-framework';
-import { retryOnDeadlock } from './deadlock.js';
-import { MSSQLConnectionManager } from '../replication/MSSQLConnectionManager.js';
+import { CaptureInstance } from '../common/CaptureInstance.js';
 import { LSN } from '../common/LSN.js';
 import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
+import { MSSQLConnectionManager } from '../replication/MSSQLConnectionManager.js';
 import { MSSQLParameter } from '../types/mssql-data-types.js';
-import * as sync_rules from '@powersync/service-sync-rules';
-import { SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
+import { retryOnDeadlock } from './deadlock.js';
 import {
   getPendingSchemaChanges,
   getReplicationIdentityColumns,
   ReplicationIdentityColumnsResult,
   ResolvedTable
 } from './schema.js';
-import * as service_types from '@powersync/service-types';
-import { CaptureInstance } from '../common/CaptureInstance.js';
 
 export const POWERSYNC_CHECKPOINTS_TABLE = '_powersync_checkpoints';
 

package/src/utils/schema.ts

@@ -1,9 +1,9 @@
+import { logger } from '@powersync/lib-services-framework';
 import { SourceEntityDescriptor } from '@powersync/service-core';
 import { TablePattern } from '@powersync/service-sync-rules';
+import sql from 'mssql';
 import { MSSQLConnectionManager } from '../replication/MSSQLConnectionManager.js';
 import { MSSQLColumnDescriptor } from '../types/mssql-data-types.js';
-import sql from 'mssql';
-import { logger } from '@powersync/lib-services-framework';
 
 export interface GetColumnsOptions {
   connectionManager: MSSQLConnectionManager;

package/test/src/CDCStream.test.ts

@@ -1,11 +1,11 @@
-import { describe, expect, test } from 'vitest';
+import { getLatestLSN } from '@module/utils/mssql.js';
+import { storage } from '@powersync/service-core';
 import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests';
 import { ReplicationMetric } from '@powersync/service-types';
-import { createTestTable, describeWithStorage, insertTestData, waitForPendingCDCChanges } from './util.js';
-import { storage } from '@powersync/service-core';
-import { CDCStreamTestContext } from './CDCStreamTestContext.js';
-import { getLatestLSN } from '@module/utils/mssql.js';
 import sql from 'mssql';
+import { describe, expect, test } from 'vitest';
+import { CDCStreamTestContext } from './CDCStreamTestContext.js';
+import { createTestTable, describeWithStorage, insertTestData, waitForPendingCDCChanges } from './util.js';
 
 const BASIC_SYNC_RULES = `
 bucket_definitions:

package/test/src/CDCStreamTestContext.ts

@@ -1,3 +1,5 @@
+import { CDCStream, CDCStreamOptions } from '@module/replication/CDCStream.js';
+import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
 import {
   BucketStorageFactory,
   createCoreReplicationMetrics,
@@ -10,10 +12,8 @@ import {
   updateSyncRulesFromYaml
 } from '@powersync/service-core';
 import { bucketRequest, METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
-import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js';
-import { CDCStream, CDCStreamOptions } from '@module/replication/CDCStream.js';
-import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
 import timers from 'timers/promises';
+import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js';
 
 /**
  * Tests operating on the change data capture need to configure the stream and manage asynchronous
@@ -156,7 +156,6 @@ export class CDCStreamTestContext implements AsyncDisposable {
       throw new Error('Call replicateSnapshot() before startStreaming()');
     }
 
-    this.cdcStream.isStartingReplication = true;
     this.streamPromise = this.cdcStream.streamChanges();
     // Wait for the replication to start before returning.
     // This avoids a bunch of unpredictable race conditions that appear in testing

package/test/src/CDCStream_resumable_snapshot.test.ts

@@ -1,13 +1,13 @@
-import { describe, expect, test } from 'vitest';
-import { env } from './env.js';
-import { createTestTableWithBasicId, describeWithStorage, waitForPendingCDCChanges } from './util.js';
-import { TestStorageConfig, TestStorageFactory } from '@powersync/service-core';
+import { getLatestLSN } from '@module/utils/mssql.js';
+import { logger, ReplicationAbortedError } from '@powersync/lib-services-framework';
+import { TestStorageConfig } from '@powersync/service-core';
 import { METRICS_HELPER } from '@powersync/service-core-tests';
 import { ReplicationMetric } from '@powersync/service-types';
 import * as timers from 'node:timers/promises';
-import { logger, ReplicationAbortedError } from '@powersync/lib-services-framework';
+import { describe, expect, test } from 'vitest';
 import { CDCStreamTestContext } from './CDCStreamTestContext.js';
-import { getLatestLSN } from '@module/utils/mssql.js';
+import { env } from './env.js';
+import { createTestTableWithBasicId, describeWithStorage, waitForPendingCDCChanges } from './util.js';
 
 describe.skipIf(!(env.CI || env.SLOW_TESTS))('batch replication', function () {
   describeWithStorage({ timeout: 240_000 }, function (config) {

package/test/src/LSN.test.ts

@@ -1,5 +1,5 @@
-import { describe, expect, test } from 'vitest';
 import { LSN } from '@module/common/LSN.js';
+import { describe, expect, test } from 'vitest';
 
 describe('LSN', () => {
   test('normalizes lowercase hex strings to uppercase', () => {

package/test/src/mssql-to-sqlite.test.ts

@@ -1,3 +1,13 @@
+import { CDCToSqliteRow, toSqliteInputRow } from '@module/common/mssqls-to-sqlite.js';
+import { CDC_SCHEMA } from '@module/common/MSSQLSourceTable.js';
+import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
+import {
+  escapeIdentifier,
+  getCaptureInstances,
+  getLatestLSN,
+  getLatestReplicatedLSN,
+  toQualifiedTableName
+} from '@module/utils/mssql.js';
 import {
   applyRowContext,
   CompatibilityContext,
@@ -5,6 +15,7 @@ import {
   SqliteInputRow,
   TimeValuePrecision
 } from '@powersync/service-sync-rules';
+import sql from 'mssql';
 import { afterAll, beforeEach, describe, expect, test } from 'vitest';
 import {
   clearTestDb,
@@ -13,17 +24,6 @@ import {
   TEST_CONNECTION_OPTIONS,
   waitForPendingCDCChanges
 } from './util.js';
-import { CDCToSqliteRow, toSqliteInputRow } from '@module/common/mssqls-to-sqlite.js';
-import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
-import {
-  escapeIdentifier,
-  getCaptureInstances,
-  getLatestLSN,
-  getLatestReplicatedLSN,
-  toQualifiedTableName
-} from '@module/utils/mssql.js';
-import sql from 'mssql';
-import { CDC_SCHEMA } from '@module/common/MSSQLSourceTable.js';
 
 describe('MSSQL Data Types Tests', () => {
   const connectionManager = new MSSQLConnectionManager(TEST_CONNECTION_OPTIONS, {});

package/test/src/schema-changes.test.ts

@@ -1,8 +1,11 @@
-import { putOp, removeOp } from '@powersync/service-core-tests';
-import { describe, expect, test, vi } from 'vitest';
 import { storage } from '@powersync/service-core';
+import { putOp, removeOp } from '@powersync/service-core-tests';
 import sql from 'mssql';
+import { describe, expect, test, vi } from 'vitest';
 
+import { SchemaChangeType } from '@module/replication/CDCPoller.js';
+import { getLatestLSN, toQualifiedTableName } from '@module/utils/mssql.js';
+import { logger } from '@powersync/lib-services-framework';
 import { CDCStreamTestContext } from './CDCStreamTestContext.js';
 import {
   createTestTableWithBasicId,
@@ -14,9 +17,6 @@ import {
   renameTable,
   waitForPendingCDCChanges
 } from './util.js';
-import { getLatestLSN, toQualifiedTableName } from '@module/utils/mssql.js';
-import { SchemaChangeType } from '@module/replication/CDCPoller.js';
-import { logger } from '@powersync/lib-services-framework';
 
 describe('MSSQL Schema Changes Tests', () => {
   describeWithStorage({ timeout: 60_000 }, defineSchemaChangesTests);
@@ -459,10 +459,14 @@ function defineSchemaChangesTests(config: storage.TestStorageConfig) {
 
 async function expectedSchemaChange(spy: any, type: SchemaChangeType) {
   logger.info(`Test Assertion: Waiting for schema change: ${type}`);
-  await vi.waitFor(() => expect(spy).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ type })), {
-    timeout: 20000
-  });
-
+  try {
+    await vi.waitFor(() => expect(spy).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ type })), {
+      timeout: 55000
+    });
+  } catch (error) {
+    // The error message thrown here is extremely verbose and not particularly helpful
+    throw new Error(`Test Assertion: Timeout waiting for schema change: ${type}`);
+  }
   const promises = spy.mock.results.filter((r: any) => r.type === 'return').map((r: any) => r.value);
 
   await Promise.all(promises.map((p: Promise<unknown>) => expect(p).resolves.toBeUndefined()));

package/test/src/util.ts

@@ -1,23 +1,17 @@
 import * as types from '@module/types/types.js';
 import { logger } from '@powersync/lib-services-framework';
-import {
-  BucketStorageFactory,
-  InternalOpId,
-  ReplicationCheckpoint,
-  TestStorageConfig,
-  TestStorageFactory
-} from '@powersync/service-core';
+import { BucketStorageFactory, InternalOpId, ReplicationCheckpoint, TestStorageConfig } from '@powersync/service-core';
 
 import * as mongo_storage from '@powersync/service-module-mongodb-storage';
 import * as postgres_storage from '@powersync/service-module-postgres-storage';
 
-import { describe, TestOptions } from 'vitest';
-import { env } from './env.js';
+import { LSN } from '@module/common/LSN.js';
 import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
 import { createCheckpoint, escapeIdentifier, getLatestLSN, toQualifiedTableName } from '@module/utils/mssql.js';
 import sql from 'mssql';
 import { v4 as uuid } from 'uuid';
-import { LSN } from '@module/common/LSN.js';
+import { describe, TestOptions } from 'vitest';
+import { env } from './env.js';
 
 export const TEST_URI = env.MSSQL_TEST_URI;