@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240918082156

package/dist/replication/AbstractReplicationJob.d.ts

@@ -0,0 +1,38 @@
+/// <reference types="node" resolution-mode="require"/>
+import * as storage from '../storage/storage-index.js';
+import { ErrorRateLimiter } from './ErrorRateLimiter.js';
+import winston from 'winston';
+export interface AbstractReplicationJobOptions {
+    id: string;
+    storage: storage.SyncRulesBucketStorage;
+    lock: storage.ReplicationLock;
+    rateLimiter: ErrorRateLimiter;
+}
+export declare abstract class AbstractReplicationJob {
+    protected options: AbstractReplicationJobOptions;
+    protected logger: winston.Logger;
+    protected abortController: AbortController;
+    protected isReplicatingPromise: Promise<void> | null;
+    protected constructor(options: AbstractReplicationJobOptions);
+    /**
+     *  Copy the initial data set from the data source if required and then keep it in sync.
+     */
+    abstract replicate(): Promise<void>;
+    /**
+     *  Ensure the connection to the data source remains active
+     */
+    abstract keepAlive(): Promise<void>;
+    /**
+     *  Start the replication process
+     */
+    start(): Promise<void>;
+    /**
+     *  Safely stop the replication process
+     */
+    stop(): Promise<void>;
+    get id(): string;
+    get storage(): storage.SyncRulesBucketStorage;
+    protected get lock(): storage.ReplicationLock;
+    protected get rateLimiter(): ErrorRateLimiter;
+    get isStopped(): boolean;
+}

package/dist/replication/AbstractReplicationJob.js

@@ -0,0 +1,51 @@
+import { container, logger } from '@powersync/lib-services-framework';
+export class AbstractReplicationJob {
+    constructor(options) {
+        this.options = options;
+        this.abortController = new AbortController();
+        this.isReplicatingPromise = null;
+        this.logger = logger.child({ name: `ReplicationJob: ${options.id}` });
+    }
+    /**
+     *  Start the replication process
+     */
+    async start() {
+        this.isReplicatingPromise = this.replicate()
+            .catch((ex) => {
+            container.reporter.captureException(ex, {
+                metadata: {
+                    replicator: this.id
+                }
+            });
+            this.logger.error(`Replication failed.`, ex);
+        })
+            .finally(async () => {
+            this.abortController.abort();
+            await this.options.lock.release();
+        });
+    }
+    /**
+     *  Safely stop the replication process
+     */
+    async stop() {
+        this.logger.info(`Stopping ${this.id} replication job for sync rule iteration: ${this.storage.group_id}`);
+        this.abortController.abort();
+        await this.isReplicatingPromise;
+    }
+    get id() {
+        return this.options.id;
+    }
+    get storage() {
+        return this.options.storage;
+    }
+    get lock() {
+        return this.options.lock;
+    }
+    get rateLimiter() {
+        return this.options.rateLimiter;
+    }
+    get isStopped() {
+        return this.abortController.signal.aborted;
+    }
+}
+//# sourceMappingURL=AbstractReplicationJob.js.map

package/dist/replication/AbstractReplicationJob.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AbstractReplicationJob.js","sourceRoot":"","sources":["../../src/replication/AbstractReplicationJob.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAUtE,MAAM,OAAgB,sBAAsB;IAK1C,YAAgC,OAAsC;QAAtC,YAAO,GAAP,OAAO,CAA+B;QAH5D,oBAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QACxC,yBAAoB,GAAyB,IAAI,CAAC;QAG1D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,mBAAmB,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACxE,CAAC;IAYD;;OAEG;IACI,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,SAAS,EAAE;aACzC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE;YACZ,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,EAAE;gBACtC,QAAQ,EAAE;oBACR,UAAU,EAAE,IAAI,CAAC,EAAE;iBACpB;aACF,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;QAC/C,CAAC,CAAC;aACD,OAAO,CAAC,KAAK,IAAI,EAAE;YAClB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACpC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,EAAE,6CAA6C,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC1G,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,oBAAoB,CAAC;IAClC,CAAC;IAED,IAAW,EAAE;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,IAAW,OAAO;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;IAC9B,CAAC;IAED,IAAc,IAAI;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;IAED,IAAc,WAAW;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAW,SAAS;QAClB,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC;IAC7C,CAAC;CACF"}

package/dist/replication/AbstractReplicator.d.ts

@@ -0,0 +1,53 @@
+import * as storage from '../storage/storage-index.js';
+import { SyncRulesProvider } from '../util/config/sync-rules/sync-rules-provider.js';
+import winston from 'winston';
+import { AbstractReplicationJob } from './AbstractReplicationJob.js';
+import { StorageEngine } from '../storage/storage-index.js';
+import { ErrorRateLimiter } from './ErrorRateLimiter.js';
+export interface CreateJobOptions {
+    lock: storage.ReplicationLock;
+    storage: storage.SyncRulesBucketStorage;
+}
+export interface AbstractReplicatorOptions {
+    id: string;
+    storageEngine: StorageEngine;
+    syncRuleProvider: SyncRulesProvider;
+    /**
+     * This limits the effect of retries when there is a persistent issue.
+     */
+    rateLimiter: ErrorRateLimiter;
+}
+/**
+ *   A replicator manages the mechanics for replicating data from a data source to a storage bucket.
+ *   This includes copying across the original data set and then keeping it in sync with the data source using Replication Jobs.
+ *   It also handles any changes to the sync rules.
+ */
+export declare abstract class AbstractReplicator<T extends AbstractReplicationJob = AbstractReplicationJob> {
+    private options;
+    protected logger: winston.Logger;
+    /**
+     *  Map of replication jobs by sync rule id. Usually there is only one running job, but there could be two when
+     *  transitioning to a new set of sync rules.
+     *  @private
+     */
+    private replicationJobs;
+    private stopped;
+    private lastPing;
+    protected constructor(options: AbstractReplicatorOptions);
+    abstract createJob(options: CreateJobOptions): T;
+    /**
+     *  Clean up any configuration or state for the specified sync rule on the datasource.
+     *  Should be a no-op if the configuration has already been cleared
+     */
+    abstract cleanUp(syncRuleStorage: storage.SyncRulesBucketStorage): Promise<void>;
+    get id(): string;
+    protected get storage(): storage.BucketStorageFactory;
+    protected get syncRuleProvider(): SyncRulesProvider;
+    protected get rateLimiter(): ErrorRateLimiter;
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    private runLoop;
+    private refresh;
+    protected createJobId(syncRuleId: number): string;
+    protected terminateSyncRules(syncRuleStorage: storage.SyncRulesBucketStorage): Promise<void>;
+}

package/dist/replication/AbstractReplicator.js

@@ -0,0 +1,187 @@
+import { hrtime } from 'node:process';
+import { container, logger } from '@powersync/lib-services-framework';
+// 5 minutes
+const PING_INTERVAL = 1000000000n * 300n;
+/**
+ *   A replicator manages the mechanics for replicating data from a data source to a storage bucket.
+ *   This includes copying across the original data set and then keeping it in sync with the data source using Replication Jobs.
+ *   It also handles any changes to the sync rules.
+ */
+export class AbstractReplicator {
+    constructor(options) {
+        this.options = options;
+        /**
+         *  Map of replication jobs by sync rule id. Usually there is only one running job, but there could be two when
+         *  transitioning to a new set of sync rules.
+         *  @private
+         */
+        this.replicationJobs = new Map();
+        this.stopped = false;
+        // First ping is only after 5 minutes, not when starting
+        this.lastPing = hrtime.bigint();
+        this.logger = logger.child({ name: `Replicator:${options.id}` });
+    }
+    get id() {
+        return this.options.id;
+    }
+    get storage() {
+        return this.options.storageEngine.activeBucketStorage;
+    }
+    get syncRuleProvider() {
+        return this.options.syncRuleProvider;
+    }
+    get rateLimiter() {
+        return this.options.rateLimiter;
+    }
+    async start() {
+        this.runLoop().catch((e) => {
+            this.logger.error('Data source fatal replication error', e);
+            container.reporter.captureException(e);
+            setTimeout(() => {
+                process.exit(1);
+            }, 1000);
+        });
+    }
+    async stop() {
+        this.stopped = true;
+        let promises = [];
+        for (const job of this.replicationJobs.values()) {
+            promises.push(job.stop());
+        }
+        await Promise.all(promises);
+    }
+    async runLoop() {
+        const syncRules = await this.syncRuleProvider.get();
+        let configuredLock = undefined;
+        if (syncRules != null) {
+            this.logger.info('Loaded sync rules');
+            try {
+                // Configure new sync rules, if they have changed.
+                // In that case, also immediately take out a lock, so that another process doesn't start replication on it.
+                const { lock } = await this.storage.configureSyncRules(syncRules, {
+                    lock: true
+                });
+                if (lock) {
+                    configuredLock = lock;
+                }
+            }
+            catch (e) {
+                // Log, but continue with previous sync rules
+                this.logger.error(`Failed to update sync rules from configuration`, e);
+            }
+        }
+        else {
+            this.logger.info('No sync rules configured - configure via API');
+        }
+        while (!this.stopped) {
+            await container.probes.touch();
+            try {
+                await this.refresh({ configured_lock: configuredLock });
+                // The lock is only valid on the first refresh.
+                configuredLock = undefined;
+                // Ensure that the replication jobs' connections are kept alive.
+                // We don't ping while in error retry back-off, to avoid having too failures.
+                if (this.rateLimiter.mayPing()) {
+                    const now = hrtime.bigint();
+                    if (now - this.lastPing >= PING_INTERVAL) {
+                        for (const activeJob of this.replicationJobs.values()) {
+                            await activeJob.keepAlive();
+                        }
+                        this.lastPing = now;
+                    }
+                }
+            }
+            catch (e) {
+                this.logger.error('Failed to refresh replication jobs', e);
+            }
+            await new Promise((resolve) => setTimeout(resolve, 5000));
+        }
+    }
+    async refresh(options) {
+        if (this.stopped) {
+            return;
+        }
+        let configuredLock = options?.configured_lock;
+        const existingJobs = new Map(this.replicationJobs.entries());
+        const replicatingSyncRules = await this.storage.getReplicatingSyncRules();
+        const newJobs = new Map();
+        for (let syncRules of replicatingSyncRules) {
+            const existingJob = existingJobs.get(syncRules.id);
+            if (existingJob && !existingJob.isStopped) {
+                // No change
+                existingJobs.delete(syncRules.id);
+                newJobs.set(syncRules.id, existingJob);
+            }
+            else if (existingJob && existingJob.isStopped) {
+                // Stopped (e.g. fatal error).
+                // Remove from the list. Next refresh call will restart the job.
+                existingJobs.delete(syncRules.id);
+            }
+            else {
+                // New sync rules were found (or resume after restart)
+                try {
+                    let lock;
+                    if (configuredLock?.sync_rules_id == syncRules.id) {
+                        lock = configuredLock;
+                    }
+                    else {
+                        lock = await syncRules.lock();
+                    }
+                    const storage = this.storage.getInstance(syncRules);
+                    const newJob = this.createJob({
+                        lock: lock,
+                        storage: storage
+                    });
+                    newJobs.set(syncRules.id, newJob);
+                    newJob.start();
+                }
+                catch (e) {
+                    // Could be a sync rules parse error,
+                    // for example from stricter validation that was added.
+                    // This will be retried every couple of seconds.
+                    // When new (valid) sync rules are deployed and processed, this one be disabled.
+                    this.logger.error('Failed to start replication for new sync rules', e);
+                }
+            }
+        }
+        this.replicationJobs = newJobs;
+        // Terminate any orphaned jobs that no longer have sync rules
+        for (let job of existingJobs.values()) {
+            // Old - stop and clean up
+            try {
+                await job.stop();
+                await this.terminateSyncRules(job.storage);
+            }
+            catch (e) {
+                // This will be retried
+                this.logger.warn('Failed to terminate old replication job}', e);
+            }
+        }
+        // Sync rules stopped previously or by a different process.
+        const stopped = await this.storage.getStoppedSyncRules();
+        for (let syncRules of stopped) {
+            try {
+                const syncRuleStorage = this.storage.getInstance(syncRules);
+                await this.terminateSyncRules(syncRuleStorage);
+            }
+            catch (e) {
+                this.logger.warn(`Failed clean up replication config for sync rule: ${syncRules.id}`, e);
+            }
+        }
+    }
+    createJobId(syncRuleId) {
+        return `${this.id}-${syncRuleId}`;
+    }
+    async terminateSyncRules(syncRuleStorage) {
+        this.logger.info(`Terminating sync rules: ${syncRuleStorage.group_id}...`);
+        try {
+            await this.cleanUp(syncRuleStorage);
+            await syncRuleStorage.terminate();
+            this.logger.info(`Successfully terminated sync rules: ${syncRuleStorage.group_id}`);
+        }
+        catch (e) {
+            this.logger.warn(`Failed clean up replication config for sync rules: ${syncRuleStorage.group_id}`, e);
+        }
+    }
+}
+//# sourceMappingURL=AbstractReplicator.js.map

package/dist/replication/AbstractReplicator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AbstractReplicator.js","sourceRoot":"","sources":["../../src/replication/AbstractReplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAOtE,YAAY;AACZ,MAAM,aAAa,GAAG,WAAc,GAAG,IAAI,CAAC;AAiB5C;;;;GAIG;AACH,MAAM,OAAgB,kBAAkB;IAatC,YAA8B,OAAkC;QAAlC,YAAO,GAAP,OAAO,CAA2B;QAXhE;;;;WAIG;QACK,oBAAe,GAAG,IAAI,GAAG,EAAa,CAAC;QACvC,YAAO,GAAG,KAAK,CAAC;QAExB,wDAAwD;QAChD,aAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QAGjC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,cAAc,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACnE,CAAC;IAUD,IAAW,EAAE;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,IAAc,OAAO;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC;IACxD,CAAC;IAED,IAAc,gBAAgB;QAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACvC,CAAC;IAED,IAAc,WAAW;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE,CAAC,CAAC,CAAC;YAC5D,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YACvC,UAAU,CAAC,GAAG,EAAE;gBACd,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,EAAE,IAAI,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,QAAQ,GAAoB,EAAE,CAAC;QACnC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE;YAC/C,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;SAC3B;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAAC;QACpD,IAAI,cAAc,GAAwC,SAAS,CAAC;QACpE,IAAI,SAAS,IAAI,IAAI,EAAE;YACrB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACtC,IAAI;gBACF,kDAAkD;gBAClD,2GAA2G;gBAC3G,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,SAAS,EAAE;oBAChE,IAAI,EAAE,IAAI;iBACX,CAAC,CAAC;gBACH,IAAI,IAAI,EAAE;oBACR,cAAc,GAAG,IAAI,CAAC;iBACvB;aACF;YAAC,OAAO,CAAC,EAAE;gBACV,6CAA6C;gBAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,EAAE,CAAC,CAAC,CAAC;aACxE;SACF;aAAM;YACL,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;SAClE;QACD,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE;YACpB,MAAM,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI;gBACF,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC;gBACxD,+CAA+C;gBAC/C,cAAc,GAAG,SAAS,CAAC;gBAE3B,gEAAgE;gBAChE,6EAA6E;gBAC7E,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE;oBAC9B,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC5B,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,IAAI,aAAa,EAAE;wBACxC,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE;4BACrD,MAAM,SAAS,CAAC,SAAS,EAAE,CAAC;yBAC7B;wBAED,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;qBACrB;iBACF;aACF;YAAC,OAAO,CAAC,EAAE;gBACV,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,EAAE,CAAC,CAAC,CAAC;aAC5D;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;SAC3D;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,OAAuD;QAC3E,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,OAAO;SACR;QAED,IAAI,cAAc,GAAG,OAAO,EAAE,eAAe,CAAC;QAE9C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAY,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,CAAC;QAC1E,MAAM,OAAO,GAAG,IAAI,GAAG,EAAa,CAAC;QACrC,KAAK,IAAI,SAAS,IAAI,oBAAoB,EAAE;YAC1C,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACnD,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE;gBACzC,YAAY;gBACZ,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBAClC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;aACxC;iBAAM,IAAI,WAAW,IAAI,WAAW,CAAC,SAAS,EAAE;gBAC/C,8BAA8B;gBAC9B,gEAAgE;gBAChE,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;aACnC;iBAAM;gBACL,sDAAsD;gBACtD,IAAI;oBACF,IAAI,IAA6B,CAAC;oBAClC,IAAI,cAAc,EAAE,aAAa,IAAI,SAAS,CAAC,EAAE,EAAE;wBACjD,IAAI,GAAG,cAAc,CAAC;qBACvB;yBAAM;wBACL,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;qBAC/B;oBACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;oBACpD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC;wBAC5B,IAAI,EAAE,IAAI;wBACV,OAAO,EAAE,OAAO;qBACjB,CAAC,CAAC;oBAEH,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;oBAClC,MAAM,CAAC,KAAK,EAAE,CAAC;iBAChB;gBAAC,OAAO,CAAC,EAAE;oBACV,qCAAqC;oBACrC,uDAAuD;oBACvD,gDAAgD;oBAChD,gFAAgF;oBAChF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,EAAE,CAAC,CAAC,CAAC;iBACxE;aACF;SACF;QAED,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;QAE/B,6DAA6D;QAC7D,KAAK,IAAI,GAAG,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE;YACrC,0BAA0B;YAC1B,IAAI;gBACF,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;aAC5C;YAAC,OAAO,CAAC,EAAE;gBACV,uBAAuB;gBACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0CAA0C,EAAE,CAAC,CAAC,CAAC;aACjE;SACF;QAED,2DAA2D;QAC3D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;QACzD,KAAK,IAAI,SAAS,IAAI,OAAO,EAAE;YAC7B,IAAI;gBACF,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;gBAC5D,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;aAChD;YAAC,OAAO,CAAC,EAAE;gBACV,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qDAAqD,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;aAC1F;SACF;IACH,CAAC;IAES,WAAW,CAAC,UAAkB;QACtC,OAAO,GAAG,IAAI,CAAC,EAAE,IAAI,UAAU,EAAE,CAAC;IACpC,CAAC;IAES,KAAK,CAAC,kBAAkB,CAAC,eAA+C;QAChF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,eAAe,CAAC,QAAQ,KAAK,CAAC,CAAC;QAC3E,IAAI;YACF,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;YACpC,MAAM,eAAe,CAAC,SAAS,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uCAAuC,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC;SACrF;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sDAAsD,eAAe,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;SACvG;IACH,CAAC;CACF"}

package/dist/replication/ErrorRateLimiter.d.ts

@@ -6,12 +6,3 @@ export interface ErrorRateLimiter {
     reportError(e: any): void;
     mayPing(): boolean;
 }
-export declare class DefaultErrorRateLimiter implements ErrorRateLimiter {
-    nextAllowed: number;
-    waitUntilAllowed(options?: {
-        signal?: AbortSignal | undefined;
-    } | undefined): Promise<void>;
-    mayPing(): boolean;
-    reportError(e: any): void;
-    private setDelay;
-}

package/dist/replication/ErrorRateLimiter.js

@@ -1,43 +1,2 @@
-import { setTimeout } from 'timers/promises';
-export class DefaultErrorRateLimiter {
-    constructor() {
-        this.nextAllowed = Date.now();
-    }
-    async waitUntilAllowed(options) {
-        const delay = Math.max(0, this.nextAllowed - Date.now());
-        // Minimum delay between connections, even without errors
-        this.setDelay(500);
-        await setTimeout(delay, undefined, { signal: options?.signal });
-    }
-    mayPing() {
-        return Date.now() >= this.nextAllowed;
-    }
-    reportError(e) {
-        const message = e.message ?? '';
-        if (message.includes('password authentication failed')) {
-            // Wait 15 minutes, to avoid triggering Supabase's fail2ban
-            this.setDelay(900000);
-        }
-        else if (message.includes('ENOTFOUND')) {
-            // DNS lookup issue - incorrect URI or deleted instance
-            this.setDelay(120000);
-        }
-        else if (message.includes('ECONNREFUSED')) {
-            // Could be fail2ban or similar
-            this.setDelay(120000);
-        }
-        else if (message.includes('Unable to do postgres query on ended pool') ||
-            message.includes('Postgres unexpectedly closed connection')) {
-            // Connection timed out - ignore / immediately retry
-            // We don't explicitly set the delay to 0, since there could have been another error that
-            // we need to respect.
-        }
-        else {
-            this.setDelay(30000);
-        }
-    }
-    setDelay(delay) {
-        this.nextAllowed = Math.max(this.nextAllowed, Date.now() + delay);
-    }
-}
+export {};
 //# sourceMappingURL=ErrorRateLimiter.js.map

package/dist/replication/ErrorRateLimiter.js.map

@@ -1 +1 @@
-{"version":3,"file":"ErrorRateLimiter.js","sourceRoot":"","sources":["../../src/replication/ErrorRateLimiter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAS7C,MAAM,OAAO,uBAAuB;IAApC;QACE,gBAAW,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;IAuCnC,CAAC;IArCC,KAAK,CAAC,gBAAgB,CAAC,OAA0D;QAC/E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACzD,yDAAyD;QACzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACnB,MAAM,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC;IACxC,CAAC;IAED,WAAW,CAAC,CAAM;QAChB,MAAM,OAAO,GAAI,CAAC,CAAC,OAAkB,IAAI,EAAE,CAAC;QAC5C,IAAI,OAAO,CAAC,QAAQ,CAAC,gCAAgC,CAAC,EAAE;YACtD,2DAA2D;YAC3D,IAAI,CAAC,QAAQ,CAAC,MAAO,CAAC,CAAC;SACxB;aAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE;YACxC,uDAAuD;YACvD,IAAI,CAAC,QAAQ,CAAC,MAAO,CAAC,CAAC;SACxB;aAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE;YAC3C,+BAA+B;YAC/B,IAAI,CAAC,QAAQ,CAAC,MAAO,CAAC,CAAC;SACxB;aAAM,IACL,OAAO,CAAC,QAAQ,CAAC,2CAA2C,CAAC;YAC7D,OAAO,CAAC,QAAQ,CAAC,yCAAyC,CAAC,EAC3D;YACA,oDAAoD;YACpD,yFAAyF;YACzF,sBAAsB;SACvB;aAAM;YACL,IAAI,CAAC,QAAQ,CAAC,KAAM,CAAC,CAAC;SACvB;IACH,CAAC;IAEO,QAAQ,CAAC,KAAa;QAC5B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;IACpE,CAAC;CACF"}
+{"version":3,"file":"ErrorRateLimiter.js","sourceRoot":"","sources":["../../src/replication/ErrorRateLimiter.ts"],"names":[],"mappings":""}

package/dist/replication/ReplicationEngine.d.ts

@@ -0,0 +1,18 @@
+import { AbstractReplicator } from './AbstractReplicator.js';
+export declare class ReplicationEngine {
+    private readonly replicators;
+    /**
+     *  Register a Replicator with the engine
+     *
+     *  @param replicator
+     */
+    register(replicator: AbstractReplicator): void;
+    /**
+     *  Start replication on all managed Replicators
+     */
+    start(): void;
+    /**
+     *  Stop replication on all managed Replicators
+     */
+    shutDown(): Promise<void>;
+}

package/dist/replication/ReplicationEngine.js

@@ -0,0 +1,41 @@
+import { logger } from '@powersync/lib-services-framework';
+export class ReplicationEngine {
+    constructor() {
+        this.replicators = new Map();
+    }
+    /**
+     *  Register a Replicator with the engine
+     *
+     *  @param replicator
+     */
+    register(replicator) {
+        if (this.replicators.has(replicator.id)) {
+            throw new Error(`Replicator with id: ${replicator.id} already registered`);
+        }
+        logger.info(`Successfully registered Replicator ${replicator.id} with ReplicationEngine`);
+        this.replicators.set(replicator.id, replicator);
+    }
+    /**
+     *  Start replication on all managed Replicators
+     */
+    start() {
+        logger.info('Starting Replication Engine...');
+        for (const replicator of this.replicators.values()) {
+            logger.info(`Starting Replicator: ${replicator.id}`);
+            replicator.start();
+        }
+        logger.info('Successfully started Replication Engine.');
+    }
+    /**
+     *  Stop replication on all managed Replicators
+     */
+    async shutDown() {
+        logger.info('Shutting down Replication Engine...');
+        for (const replicator of this.replicators.values()) {
+            logger.info(`Stopping Replicator: ${replicator.id}`);
+            await replicator.stop();
+        }
+        logger.info('Successfully shut down Replication Engine.');
+    }
+}
+//# sourceMappingURL=ReplicationEngine.js.map

package/dist/replication/ReplicationEngine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ReplicationEngine.js","sourceRoot":"","sources":["../../src/replication/ReplicationEngine.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,MAAM,OAAO,iBAAiB;IAA9B;QACmB,gBAAW,GAAoC,IAAI,GAAG,EAAE,CAAC;IAsC5E,CAAC;IApCC;;;;OAIG;IACI,QAAQ,CAAC,UAA8B;QAC5C,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,EAAE;YACvC,MAAM,IAAI,KAAK,CAAC,uBAAuB,UAAU,CAAC,EAAE,qBAAqB,CAAC,CAAC;SAC5E;QACD,MAAM,CAAC,IAAI,CAAC,sCAAsC,UAAU,CAAC,EAAE,yBAAyB,CAAC,CAAC;QAC1F,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACI,KAAK;QACV,MAAM,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;QAC9C,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE;YAClD,MAAM,CAAC,IAAI,CAAC,wBAAwB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;YACrD,UAAU,CAAC,KAAK,EAAE,CAAC;SACpB;QACD,MAAM,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,QAAQ;QACnB,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;QACnD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE;YAClD,MAAM,CAAC,IAAI,CAAC,wBAAwB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;YACrD,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;SACzB;QACD,MAAM,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;IAC5D,CAAC;CACF"}

package/dist/replication/ReplicationModule.d.ts

@@ -0,0 +1,39 @@
+import { DataSourceConfig } from '@powersync/service-types/dist/config/PowerSyncConfig.js';
+import * as t from 'ts-codec';
+import * as api from '../api/api-index.js';
+import * as modules from '../modules/modules-index.js';
+import * as system from '../system/system-index.js';
+import { AbstractReplicator } from './AbstractReplicator.js';
+export interface ReplicationModuleOptions extends modules.AbstractModuleOptions {
+    type: string;
+    configSchema: t.AnyCodec;
+}
+/**
+ *  A replication module describes all the functionality that PowerSync requires to
+ *  replicate data from a DataSource. Whenever a new data source is added to powersync this class should be extended.
+ */
+export declare abstract class ReplicationModule<TConfig extends DataSourceConfig> extends modules.AbstractModule {
+    protected type: string;
+    protected configSchema: t.AnyCodec;
+    protected decodedConfig: TConfig | undefined;
+    /**
+     * @protected
+     * @param options
+     */
+    protected constructor(options: ReplicationModuleOptions);
+    /**
+     *  Create the RouteAPI adapter for the DataSource required to service the sync API
+     *  endpoints.
+     */
+    protected abstract createRouteAPIAdapter(): api.RouteAPI;
+    /**
+     *  Create the Replicator to be used by the ReplicationEngine.
+     */
+    protected abstract createReplicator(context: system.ServiceContext): AbstractReplicator;
+    /**
+     *  Register this module's Replicators and RouteAPI adapters if the required configuration is present.
+     */
+    initialize(context: system.ServiceContext): Promise<void>;
+    private validateConfig;
+    protected getDefaultId(dataSourceName: string): string;
+}

package/dist/replication/ReplicationModule.js

@@ -0,0 +1,65 @@
+import * as t from 'ts-codec';
+import * as types from '@powersync/service-types';
+import * as modules from '../modules/modules-index.js';
+import { schema } from '@powersync/lib-services-framework';
+/**
+ *  A replication module describes all the functionality that PowerSync requires to
+ *  replicate data from a DataSource. Whenever a new data source is added to powersync this class should be extended.
+ */
+export class ReplicationModule extends modules.AbstractModule {
+    /**
+     * @protected
+     * @param options
+     */
+    constructor(options) {
+        super(options);
+        this.type = options.type;
+        this.configSchema = options.configSchema;
+    }
+    /**
+     *  Register this module's Replicators and RouteAPI adapters if the required configuration is present.
+     */
+    async initialize(context) {
+        if (!context.configuration.connections) {
+            // No data source configuration found in the config skip for now
+            return;
+        }
+        const matchingConfig = context.configuration.connections.filter((dataSource) => dataSource.type === this.type);
+        if (!matchingConfig.length) {
+            // This module is needed given the config
+            return;
+        }
+        if (matchingConfig.length > 1) {
+            this.logger.warning(`Multiple data sources of type ${this.type} found in the configuration. Only the first will be used.`);
+        }
+        try {
+            const baseMatchingConfig = matchingConfig[0];
+            // If validation fails, log the error and continue, no replication will happen for this data source
+            this.validateConfig(baseMatchingConfig);
+            this.decodedConfig = this.configSchema.decode(baseMatchingConfig);
+            context.replicationEngine?.register(this.createReplicator(context));
+            context.routerEngine?.registerAPI(this.createRouteAPIAdapter());
+        }
+        catch (e) {
+            this.logger.error('Failed to initialize.', e);
+        }
+    }
+    validateConfig(config) {
+        const validator = schema
+            .parseJSONSchema(
+        // This generates a schema for the encoded form of the codec
+        t.generateJSONSchema(this.configSchema, {
+            allowAdditional: true,
+            parsers: [types.configFile.portParser]
+        }))
+            .validator();
+        const valid = validator.validate(config);
+        if (!valid.valid) {
+            throw new Error(`Failed to validate Module ${this.name} configuration: ${valid.errors.join(', ')}`);
+        }
+    }
+    getDefaultId(dataSourceName) {
+        return `${this.type}-${dataSourceName}`;
+    }
+}
+//# sourceMappingURL=ReplicationModule.js.map

package/dist/replication/ReplicationModule.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ReplicationModule.js","sourceRoot":"","sources":["../../src/replication/ReplicationModule.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,OAAO,KAAK,KAAK,MAAM,0BAA0B,CAAC;AAElD,OAAO,KAAK,OAAO,MAAM,6BAA6B,CAAC;AAEvD,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAQ3D;;;GAGG;AACH,MAAM,OAAgB,iBAAoD,SAAQ,OAAO,CAAC,cAAc;IAKtG;;;OAGG;IACH,YAAsB,OAAiC;QACrD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAC3C,CAAC;IAaD;;OAEG;IACI,KAAK,CAAC,UAAU,CAAC,OAA8B;QACpD,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,WAAW,EAAE;YACtC,gEAAgE;YAChE,OAAO;SACR;QAED,MAAM,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/G,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE;YAC1B,yCAAyC;YACzC,OAAO;SACR;QAED,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;YAC7B,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,iCAAiC,IAAI,CAAC,IAAI,2DAA2D,CACtG,CAAC;SACH;QAED,IAAI;YACF,MAAM,kBAAkB,GAAG,cAAc,CAAC,CAAC,CAAY,CAAC;YACxD,mGAAmG;YACnG,IAAI,CAAC,cAAc,CAAC,kBAAkB,CAAC,CAAC;YACxC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;YAElE,OAAO,CAAC,iBAAiB,EAAE,QAAQ,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;YACpE,OAAO,CAAC,YAAY,EAAE,WAAW,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC,CAAC;SACjE;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC;SAC/C;IACH,CAAC;IAEO,cAAc,CAAC,MAAe;QACpC,MAAM,SAAS,GAAG,MAAM;aACrB,eAAe;QACd,4DAA4D;QAC5D,CAAC,CAAC,kBAAkB,CAAC,IAAI,CAAC,YAAY,EAAE;YACtC,eAAe,EAAE,IAAI;YACrB,OAAO,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC;SACvC,CAAC,CACH;aACA,SAAS,EAAE,CAAC;QAEf,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEzC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE;YAChB,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,CAAC,IAAI,mBAAmB,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACrG;IACH,CAAC;IAES,YAAY,CAAC,cAAsB;QAC3C,OAAO,GAAG,IAAI,CAAC,IAAI,IAAI,cAAc,EAAE,CAAC;IAC1C,CAAC;CACF"}

package/dist/replication/replication-index.d.ts

@@ -1,7 +1,5 @@
 export * from './ErrorRateLimiter.js';
-export * from './PgRelation.js';
-export * from './util.js';
-export * from './WalConnection.js';
-export * from './WalStream.js';
-export * from './WalStreamManager.js';
-export * from './WalStreamRunner.js';
+export * from './AbstractReplicationJob.js';
+export * from './AbstractReplicator.js';
+export * from './ReplicationEngine.js';
+export * from './ReplicationModule.js';

package/dist/replication/replication-index.js

@@ -1,8 +1,6 @@
 export * from './ErrorRateLimiter.js';
-export * from './PgRelation.js';
-export * from './util.js';
-export * from './WalConnection.js';
-export * from './WalStream.js';
-export * from './WalStreamManager.js';
-export * from './WalStreamRunner.js';
+export * from './AbstractReplicationJob.js';
+export * from './AbstractReplicator.js';
+export * from './ReplicationEngine.js';
+export * from './ReplicationModule.js';
 //# sourceMappingURL=replication-index.js.map

package/dist/replication/replication-index.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-index.js","sourceRoot":"","sources":["../../src/replication/replication-index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC;AACtC,cAAc,iBAAiB,CAAC;AAChC,cAAc,WAAW,CAAC;AAC1B,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC"}
+{"version":3,"file":"replication-index.js","sourceRoot":"","sources":["../../src/replication/replication-index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC;AACtC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC"}