@powersync/service-module-mssql 0.0.1

package/dist/replication/CDCReplicator.js

@@ -0,0 +1,55 @@
+import { replication } from '@powersync/service-core';
+import { CDCReplicationJob } from './CDCReplicationJob.js';
+import { MSSQLModule } from '../module/MSSQLModule.js';
+export class CDCReplicator extends replication.AbstractReplicator {
+    connectionFactory;
+    cdcReplicatorOptions;
+    constructor(options) {
+        super(options);
+        this.connectionFactory = options.connectionFactory;
+        this.cdcReplicatorOptions = options;
+    }
+    createJob(options) {
+        return new CDCReplicationJob({
+            id: this.createJobId(options.storage.group_id),
+            storage: options.storage,
+            metrics: this.metrics,
+            lock: options.lock,
+            connectionFactory: this.connectionFactory,
+            rateLimiter: this.rateLimiter,
+            pollingOptions: this.cdcReplicatorOptions.pollingOptions
+        });
+    }
+    async cleanUp(syncRulesStorage) { }
+    async stop() {
+        await super.stop();
+        await this.connectionFactory.shutdown();
+    }
+    async testConnection() {
+        return await MSSQLModule.testConnection(this.connectionFactory.connectionConfig);
+    }
+    async getReplicationLagMillis() {
+        // TODO:Get replication lag
+        const lag = await super.getReplicationLagMillis();
+        if (lag != null) {
+            return lag;
+        }
+        // Booting or in an error loop. Check last active replication status.
+        // This includes sync rules in an ERROR state.
+        const content = await this.storage.getActiveSyncRulesContent();
+        if (content == null) {
+            return undefined;
+        }
+        // Measure the lag from the last commit or keepalive timestamp.
+        // This is not 100% accurate since it is the commit time in the storage db rather than
+        // the source db, but it's the best we currently have for mssql.
+        const checkpointTs = content.last_checkpoint_ts?.getTime() ?? 0;
+        const keepaliveTs = content.last_keepalive_ts?.getTime() ?? 0;
+        const latestTs = Math.max(checkpointTs, keepaliveTs);
+        if (latestTs != 0) {
+            return Date.now() - latestTs;
+        }
+        return undefined;
+    }
+}
+//# sourceMappingURL=CDCReplicator.js.map

package/dist/replication/CDCReplicator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CDCReplicator.js","sourceRoot":"","sources":["../../src/replication/CDCReplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAW,MAAM,yBAAyB,CAAC;AAE/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAQvD,MAAM,OAAO,aAAc,SAAQ,WAAW,CAAC,kBAAqC;IACjE,iBAAiB,CAAgC;IACjD,oBAAoB,CAAuB;IAE5D,YAAY,OAA6B;QACvC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC;IACtC,CAAC;IAED,SAAS,CAAC,OAAqC;QAC7C,OAAO,IAAI,iBAAiB,CAAC;YAC3B,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC;YAC9C,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,cAAc,EAAE,IAAI,CAAC,oBAAoB,CAAC,cAAc;SACzD,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,gBAAgD,IAAkB,CAAC;IAEjF,KAAK,CAAC,IAAI;QACR,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,cAAc;QAClB,OAAO,MAAM,WAAW,CAAC,cAAc,CAAC,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,CAAC;IACnF,CAAC;IAED,KAAK,CAAC,uBAAuB;QAC3B,2BAA2B;QAC3B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,uBAAuB,EAAE,CAAC;QAClD,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,OAAO,GAAG,CAAC;QACb,CAAC;QAED,qEAAqE;QACrE,8CAA8C;QAC9C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,yBAAyB,EAAE,CAAC;QAC/D,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,+DAA+D;QAC/D,sFAAsF;QACtF,gEAAgE;QAChE,MAAM,YAAY,GAAG,OAAO,CAAC,kBAAkB,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;QAChE,MAAM,WAAW,GAAG,OAAO,CAAC,iBAAiB,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;QAC9D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;QACrD,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC;QAC/B,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;CACF"}

package/dist/replication/CDCStream.d.ts

@@ -0,0 +1,106 @@
+import { DatabaseConnectionError, Logger } from '@powersync/lib-services-framework';
+import { MetricsEngine, SourceEntityDescriptor, storage } from '@powersync/service-core';
+import { TablePattern } from '@powersync/service-sync-rules';
+import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
+import sql from 'mssql';
+import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
+import { CDCPollingOptions } from '../types/types.js';
+export interface CDCStreamOptions {
+    connections: MSSQLConnectionManager;
+    storage: storage.SyncRulesBucketStorage;
+    metrics: MetricsEngine;
+    abortSignal: AbortSignal;
+    logger?: Logger;
+    /**
+     * Override snapshot batch size for testing.
+     * Defaults to 10_000.
+     * Note that queries are streamed, so we don't keep that much data in memory.
+     */
+    snapshotBatchSize?: number;
+    pollingOptions: CDCPollingOptions;
+}
+export declare enum SnapshotStatus {
+    IN_PROGRESS = "in-progress",
+    DONE = "done",
+    RESTART_REQUIRED = "restart-required"
+}
+export interface SnapshotStatusResult {
+    status: SnapshotStatus;
+    snapshotLSN: string | null;
+}
+export declare class CDCConfigurationError extends Error {
+    constructor(message: string);
+}
+/**
+ * Thrown when required updates in the CDC instance tables are no longer available
+ *
+ * Possible reasons:
+ *  * Older data has been cleaned up due to exceeding the retention period.
+ *    This can happen if PowerSync was stopped for a long period of time.
+ */
+export declare class CDCDataExpiredError extends DatabaseConnectionError {
+    constructor(message: string, cause: any);
+}
+export declare class CDCStream {
+    private options;
+    private readonly syncRules;
+    private readonly storage;
+    private readonly connections;
+    private readonly abortSignal;
+    private readonly logger;
+    private tableCache;
+    /**
+     * Time of the oldest uncommitted change, according to the source db.
+     * This is used to determine the replication lag.
+     */
+    private oldestUncommittedChange;
+    /**
+     * Keep track of whether we have done a commit or keepalive yet.
+     * We can only compute replication lag if isStartingReplication == false, or oldestUncommittedChange is present.
+     */
+    isStartingReplication: boolean;
+    constructor(options: CDCStreamOptions);
+    private get metrics();
+    get stopped(): boolean;
+    get defaultSchema(): string;
+    get groupId(): number;
+    get connectionId(): number;
+    get connectionTag(): string;
+    get snapshotBatchSize(): number;
+    replicate(): Promise<void>;
+    populateTableCache(): Promise<void>;
+    getQualifiedTableNames(batch: storage.BucketStorageBatch, tablePattern: TablePattern): Promise<MSSQLSourceTable[]>;
+    processTable(batch: storage.BucketStorageBatch, table: SourceEntityDescriptor, snapshot: boolean): Promise<MSSQLSourceTable>;
+    private snapshotTableInTx;
+    private snapshotTable;
+    /**
+     *  Estimate the number of rows in a table. This query uses partition stats view to get a fast estimate of the row count.
+     *  This requires that the MSSQL DB user has the VIEW DATABASE PERFORMANCE STATE permission.
+     * @param table
+     * @param transaction
+     */
+    estimatedCountNumber(table: MSSQLSourceTable, transaction?: sql.Transaction): Promise<number>;
+    /**
+     * Start initial replication.
+     *
+     * If (partial) replication was done before on this slot, this clears the state
+     * and starts again from scratch.
+     */
+    startInitialReplication(snapshotStatus: SnapshotStatusResult): Promise<void>;
+    initReplication(): Promise<void>;
+    /**
+     * Checks if the initial sync has already been completed and if updates from the last checkpoint are still available
+     * in the CDC instances.
+     */
+    private checkSnapshotStatus;
+    streamChanges(): Promise<void>;
+    private createEventHandler;
+    /**
+     * Convert CDC row data to SqliteRow format.
+     * CDC rows include table columns plus CDC metadata columns (__$operation, __$start_lsn, etc.).
+     * We filter out the CDC metadata columns.
+     */
+    private toSqliteRow;
+    getReplicationLagMillis(): Promise<number | undefined>;
+    private touch;
+}