@powersync/service-core 0.8.8 → 0.10.0

package/dist/replication/AbstractReplicator.js

@@ -0,0 +1,250 @@
+var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
+    if (value !== null && value !== void 0) {
+        if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
+        var dispose, inner;
+        if (async) {
+            if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
+            dispose = value[Symbol.asyncDispose];
+        }
+        if (dispose === void 0) {
+            if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
+            dispose = value[Symbol.dispose];
+            if (async) inner = dispose;
+        }
+        if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
+        if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
+        env.stack.push({ value: value, dispose: dispose, async: async });
+    }
+    else if (async) {
+        env.stack.push({ async: true });
+    }
+    return value;
+};
+var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
+    return function (env) {
+        function fail(e) {
+            env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
+            env.hasError = true;
+        }
+        var r, s = 0;
+        function next() {
+            while (r = env.stack.pop()) {
+                try {
+                    if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
+                    if (r.dispose) {
+                        var result = r.dispose.call(r.value);
+                        if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
+                    }
+                    else s |= 1;
+                }
+                catch (e) {
+                    fail(e);
+                }
+            }
+            if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
+            if (env.hasError) throw env.error;
+        }
+        return next();
+    };
+})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
+    var e = new Error(message);
+    return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
+});
+import { container, logger } from '@powersync/lib-services-framework';
+import { hrtime } from 'node:process';
+// 5 minutes
+const PING_INTERVAL = 1000000000n * 300n;
+/**
+ *   A replicator manages the mechanics for replicating data from a data source to a storage bucket.
+ *   This includes copying across the original data set and then keeping it in sync with the data source using Replication Jobs.
+ *   It also handles any changes to the sync rules.
+ */
+export class AbstractReplicator {
+    constructor(options) {
+        this.options = options;
+        /**
+         *  Map of replication jobs by sync rule id. Usually there is only one running job, but there could be two when
+         *  transitioning to a new set of sync rules.
+         *  @private
+         */
+        this.replicationJobs = new Map();
+        this.stopped = false;
+        // First ping is only after 5 minutes, not when starting
+        this.lastPing = hrtime.bigint();
+        this.logger = logger.child({ name: `Replicator:${options.id}` });
+    }
+    get id() {
+        return this.options.id;
+    }
+    get storage() {
+        return this.options.storageEngine.activeBucketStorage;
+    }
+    get syncRuleProvider() {
+        return this.options.syncRuleProvider;
+    }
+    get rateLimiter() {
+        return this.options.rateLimiter;
+    }
+    async start() {
+        this.runLoop().catch((e) => {
+            this.logger.error('Data source fatal replication error', e);
+            container.reporter.captureException(e);
+            setTimeout(() => {
+                process.exit(1);
+            }, 1000);
+        });
+    }
+    async stop() {
+        this.stopped = true;
+        let promises = [];
+        for (const job of this.replicationJobs.values()) {
+            promises.push(job.stop());
+        }
+        await Promise.all(promises);
+    }
+    async runLoop() {
+        const syncRules = await this.syncRuleProvider.get();
+        let configuredLock = undefined;
+        if (syncRules != null) {
+            this.logger.info('Loaded sync rules');
+            try {
+                // Configure new sync rules, if they have changed.
+                // In that case, also immediately take out a lock, so that another process doesn't start replication on it.
+                const { lock } = await this.storage.configureSyncRules(syncRules, {
+                    lock: true
+                });
+                if (lock) {
+                    configuredLock = lock;
+                }
+            }
+            catch (e) {
+                // Log, but continue with previous sync rules
+                this.logger.error(`Failed to update sync rules from configuration`, e);
+            }
+        }
+        else {
+            this.logger.info('No sync rules configured - configure via API');
+        }
+        while (!this.stopped) {
+            await container.probes.touch();
+            try {
+                await this.refresh({ configured_lock: configuredLock });
+                // The lock is only valid on the first refresh.
+                configuredLock = undefined;
+                // Ensure that the replication jobs' connections are kept alive.
+                // We don't ping while in error retry back-off, to avoid having too failures.
+                if (this.rateLimiter.mayPing()) {
+                    const now = hrtime.bigint();
+                    if (now - this.lastPing >= PING_INTERVAL) {
+                        for (const activeJob of this.replicationJobs.values()) {
+                            await activeJob.keepAlive();
+                        }
+                        this.lastPing = now;
+                    }
+                }
+            }
+            catch (e) {
+                this.logger.error('Failed to refresh replication jobs', e);
+            }
+            await new Promise((resolve) => setTimeout(resolve, 5000));
+        }
+    }
+    async refresh(options) {
+        if (this.stopped) {
+            return;
+        }
+        let configuredLock = options?.configured_lock;
+        const existingJobs = new Map(this.replicationJobs.entries());
+        const replicatingSyncRules = await this.storage.getReplicatingSyncRules();
+        const newJobs = new Map();
+        for (let syncRules of replicatingSyncRules) {
+            const existingJob = existingJobs.get(syncRules.id);
+            if (existingJob && !existingJob.isStopped) {
+                // No change
+                existingJobs.delete(syncRules.id);
+                newJobs.set(syncRules.id, existingJob);
+            }
+            else if (existingJob && existingJob.isStopped) {
+                // Stopped (e.g. fatal error).
+                // Remove from the list. Next refresh call will restart the job.
+                existingJobs.delete(syncRules.id);
+            }
+            else {
+                // New sync rules were found (or resume after restart)
+                try {
+                    let lock;
+                    if (configuredLock?.sync_rules_id == syncRules.id) {
+                        lock = configuredLock;
+                    }
+                    else {
+                        lock = await syncRules.lock();
+                    }
+                    const storage = this.storage.getInstance(syncRules);
+                    const newJob = this.createJob({
+                        lock: lock,
+                        storage: storage
+                    });
+                    newJobs.set(syncRules.id, newJob);
+                    newJob.start();
+                }
+                catch (e) {
+                    // Could be a sync rules parse error,
+                    // for example from stricter validation that was added.
+                    // This will be retried every couple of seconds.
+                    // When new (valid) sync rules are deployed and processed, this one be disabled.
+                    this.logger.error('Failed to start replication for new sync rules', e);
+                }
+            }
+        }
+        this.replicationJobs = newJobs;
+        // Terminate any orphaned jobs that no longer have sync rules
+        for (let job of existingJobs.values()) {
+            // Old - stop and clean up
+            try {
+                await job.stop();
+                await this.terminateSyncRules(job.storage);
+                job.storage[Symbol.dispose]();
+            }
+            catch (e) {
+                // This will be retried
+                this.logger.warn('Failed to terminate old replication job}', e);
+            }
+        }
+        // Sync rules stopped previously or by a different process.
+        const stopped = await this.storage.getStoppedSyncRules();
+        for (let syncRules of stopped) {
+            try {
+                const env_1 = { stack: [], error: void 0, hasError: false };
+                try {
+                    const syncRuleStorage = __addDisposableResource(env_1, this.storage.getInstance(syncRules), false);
+                    await this.terminateSyncRules(syncRuleStorage);
+                }
+                catch (e_1) {
+                    env_1.error = e_1;
+                    env_1.hasError = true;
+                }
+                finally {
+                    __disposeResources(env_1);
+                }
+            }
+            catch (e) {
+                this.logger.warn(`Failed clean up replication config for sync rule: ${syncRules.id}`, e);
+            }
+        }
+    }
+    createJobId(syncRuleId) {
+        return `${this.id}-${syncRuleId}`;
+    }
+    async terminateSyncRules(syncRuleStorage) {
+        this.logger.info(`Terminating sync rules: ${syncRuleStorage.group_id}...`);
+        try {
+            await this.cleanUp(syncRuleStorage);
+            await syncRuleStorage.terminate();
+            this.logger.info(`Successfully terminated sync rules: ${syncRuleStorage.group_id}`);
+        }
+        catch (e) {
+            this.logger.warn(`Failed clean up replication config for sync rules: ${syncRuleStorage.group_id}`, e);
+        }
+    }
+}
+//# sourceMappingURL=AbstractReplicator.js.map

package/dist/replication/AbstractReplicator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AbstractReplicator.js","sourceRoot":"","sources":["../../src/replication/AbstractReplicator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAQtC,YAAY;AACZ,MAAM,aAAa,GAAG,WAAc,GAAG,IAAI,CAAC;AAiB5C;;;;GAIG;AACH,MAAM,OAAgB,kBAAkB;IAatC,YAA8B,OAAkC;QAAlC,YAAO,GAAP,OAAO,CAA2B;QAXhE;;;;WAIG;QACK,oBAAe,GAAG,IAAI,GAAG,EAAa,CAAC;QACvC,YAAO,GAAG,KAAK,CAAC;QAExB,wDAAwD;QAChD,aAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QAGjC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,cAAc,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACnE,CAAC;IAUD,IAAW,EAAE;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,IAAc,OAAO;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC;IACxD,CAAC;IAED,IAAc,gBAAgB;QAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACvC,CAAC;IAED,IAAc,WAAW;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE,CAAC,CAAC,CAAC;YAC5D,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YACvC,UAAU,CAAC,GAAG,EAAE;gBACd,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,EAAE,IAAI,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,QAAQ,GAAoB,EAAE,CAAC;QACnC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;YAChD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5B,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAAC;QACpD,IAAI,cAAc,GAAwC,SAAS,CAAC;QACpE,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACtC,IAAI,CAAC;gBACH,kDAAkD;gBAClD,2GAA2G;gBAC3G,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,SAAS,EAAE;oBAChE,IAAI,EAAE,IAAI;iBACX,CAAC,CAAC;gBACH,IAAI,IAAI,EAAE,CAAC;oBACT,cAAc,GAAG,IAAI,CAAC;gBACxB,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,6CAA6C;gBAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,EAAE,CAAC,CAAC,CAAC;YACzE,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;QACnE,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC;gBACxD,+CAA+C;gBAC/C,cAAc,GAAG,SAAS,CAAC;gBAE3B,gEAAgE;gBAChE,6EAA6E;gBAC7E,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;oBAC/B,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC5B,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,IAAI,aAAa,EAAE,CAAC;wBACzC,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;4BACtD,MAAM,SAAS,CAAC,SAAS,EAAE,CAAC;wBAC9B,CAAC;wBAED,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;oBACtB,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,EAAE,CAAC,CAAC,CAAC;YAC7D,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,OAAuD;QAC3E,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,IAAI,cAAc,GAAG,OAAO,EAAE,eAAe,CAAC;QAE9C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAY,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,CAAC;QAC1E,MAAM,OAAO,GAAG,IAAI,GAAG,EAAa,CAAC;QACrC,KAAK,IAAI,SAAS,IAAI,oBAAoB,EAAE,CAAC;YAC3C,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACnD,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;gBAC1C,YAAY;gBACZ,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBAClC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;YACzC,CAAC;iBAAM,IAAI,WAAW,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;gBAChD,8BAA8B;gBAC9B,gEAAgE;gBAChE,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,sDAAsD;gBACtD,IAAI,CAAC;oBACH,IAAI,IAA6B,CAAC;oBAClC,IAAI,cAAc,EAAE,aAAa,IAAI,SAAS,CAAC,EAAE,EAAE,CAAC;wBAClD,IAAI,GAAG,cAAc,CAAC;oBACxB,CAAC;yBAAM,CAAC;wBACN,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;oBAChC,CAAC;oBACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;oBACpD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC;wBAC5B,IAAI,EAAE,IAAI;wBACV,OAAO,EAAE,OAAO;qBACjB,CAAC,CAAC;oBAEH,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;oBAClC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,qCAAqC;oBACrC,uDAAuD;oBACvD,gDAAgD;oBAChD,gFAAgF;oBAChF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,EAAE,CAAC,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;QAE/B,6DAA6D;QAC7D,KAAK,IAAI,GAAG,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;YACtC,0BAA0B;YAC1B,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;gBACjB,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAC3C,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YAChC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,uBAAuB;gBACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0CAA0C,EAAE,CAAC,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;QAED,2DAA2D;QAC3D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;QACzD,KAAK,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,CAAC;;;oBACH,MAAM,eAAe,kCAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,QAAA,CAAC;oBAC5D,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;;;;;;;;;aAChD;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qDAAqD,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;IACH,CAAC;IAES,WAAW,CAAC,UAAkB;QACtC,OAAO,GAAG,IAAI,CAAC,EAAE,IAAI,UAAU,EAAE,CAAC;IACpC,CAAC;IAES,KAAK,CAAC,kBAAkB,CAAC,eAA+C;QAChF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,eAAe,CAAC,QAAQ,KAAK,CAAC,CAAC;QAC3E,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;YACpC,MAAM,eAAe,CAAC,SAAS,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uCAAuC,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC;QACtF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sDAAsD,eAAe,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;QACxG,CAAC;IACH,CAAC;CACF"}

package/dist/replication/ErrorRateLimiter.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" resolution-mode="require"/>
 export interface ErrorRateLimiter {
     waitUntilAllowed(options?: {
         signal?: AbortSignal;
@@ -6,12 +5,3 @@ export interface ErrorRateLimiter {
     reportError(e: any): void;
     mayPing(): boolean;
 }
-export declare class DefaultErrorRateLimiter implements ErrorRateLimiter {
-    nextAllowed: number;
-    waitUntilAllowed(options?: {
-        signal?: AbortSignal | undefined;
-    } | undefined): Promise<void>;
-    mayPing(): boolean;
-    reportError(e: any): void;
-    private setDelay;
-}

package/dist/replication/ErrorRateLimiter.js

@@ -1,43 +1,2 @@
-import { setTimeout } from 'timers/promises';
-export class DefaultErrorRateLimiter {
-    constructor() {
-        this.nextAllowed = Date.now();
-    }
-    async waitUntilAllowed(options) {
-        const delay = Math.max(0, this.nextAllowed - Date.now());
-        // Minimum delay between connections, even without errors
-        this.setDelay(500);
-        await setTimeout(delay, undefined, { signal: options?.signal });
-    }
-    mayPing() {
-        return Date.now() >= this.nextAllowed;
-    }
-    reportError(e) {
-        const message = e.message ?? '';
-        if (message.includes('password authentication failed')) {
-            // Wait 15 minutes, to avoid triggering Supabase's fail2ban
-            this.setDelay(900000);
-        }
-        else if (message.includes('ENOTFOUND')) {
-            // DNS lookup issue - incorrect URI or deleted instance
-            this.setDelay(120000);
-        }
-        else if (message.includes('ECONNREFUSED')) {
-            // Could be fail2ban or similar
-            this.setDelay(120000);
-        }
-        else if (message.includes('Unable to do postgres query on ended pool') ||
-            message.includes('Postgres unexpectedly closed connection')) {
-            // Connection timed out - ignore / immediately retry
-            // We don't explicitly set the delay to 0, since there could have been another error that
-            // we need to respect.
-        }
-        else {
-            this.setDelay(30000);
-        }
-    }
-    setDelay(delay) {
-        this.nextAllowed = Math.max(this.nextAllowed, Date.now() + delay);
-    }
-}
+export {};
 //# sourceMappingURL=ErrorRateLimiter.js.map

package/dist/replication/ErrorRateLimiter.js.map

@@ -1 +1 @@
-{"version":3,"file":"ErrorRateLimiter.js","sourceRoot":"","sources":["../../src/replication/ErrorRateLimiter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAS7C,MAAM,OAAO,uBAAuB;IAApC;QACE,gBAAW,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;IAuCnC,CAAC;IArCC,KAAK,CAAC,gBAAgB,CAAC,OAA0D;QAC/E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACzD,yDAAyD;QACzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACnB,MAAM,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC;IACxC,CAAC;IAED,WAAW,CAAC,CAAM;QAChB,MAAM,OAAO,GAAI,CAAC,CAAC,OAAkB,IAAI,EAAE,CAAC;QAC5C,IAAI,OAAO,CAAC,QAAQ,CAAC,gCAAgC,CAAC,EAAE;YACtD,2DAA2D;YAC3D,IAAI,CAAC,QAAQ,CAAC,MAAO,CAAC,CAAC;SACxB;aAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE;YACxC,uDAAuD;YACvD,IAAI,CAAC,QAAQ,CAAC,MAAO,CAAC,CAAC;SACxB;aAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE;YAC3C,+BAA+B;YAC/B,IAAI,CAAC,QAAQ,CAAC,MAAO,CAAC,CAAC;SACxB;aAAM,IACL,OAAO,CAAC,QAAQ,CAAC,2CAA2C,CAAC;YAC7D,OAAO,CAAC,QAAQ,CAAC,yCAAyC,CAAC,EAC3D;YACA,oDAAoD;YACpD,yFAAyF;YACzF,sBAAsB;SACvB;aAAM;YACL,IAAI,CAAC,QAAQ,CAAC,KAAM,CAAC,CAAC;SACvB;IACH,CAAC;IAEO,QAAQ,CAAC,KAAa;QAC5B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;IACpE,CAAC;CACF"}
+{"version":3,"file":"ErrorRateLimiter.js","sourceRoot":"","sources":["../../src/replication/ErrorRateLimiter.ts"],"names":[],"mappings":""}

package/dist/replication/ReplicationEngine.d.ts

@@ -0,0 +1,18 @@
+import { AbstractReplicator } from './AbstractReplicator.js';
+export declare class ReplicationEngine {
+    private readonly replicators;
+    /**
+     *  Register a Replicator with the engine
+     *
+     *  @param replicator
+     */
+    register(replicator: AbstractReplicator): void;
+    /**
+     *  Start replication on all managed Replicators
+     */
+    start(): void;
+    /**
+     *  Stop replication on all managed Replicators
+     */
+    shutDown(): Promise<void>;
+}

package/dist/replication/ReplicationEngine.js

@@ -0,0 +1,41 @@
+import { logger } from '@powersync/lib-services-framework';
+export class ReplicationEngine {
+    constructor() {
+        this.replicators = new Map();
+    }
+    /**
+     *  Register a Replicator with the engine
+     *
+     *  @param replicator
+     */
+    register(replicator) {
+        if (this.replicators.has(replicator.id)) {
+            throw new Error(`Replicator with id: ${replicator.id} already registered`);
+        }
+        logger.info(`Successfully registered Replicator ${replicator.id} with ReplicationEngine`);
+        this.replicators.set(replicator.id, replicator);
+    }
+    /**
+     *  Start replication on all managed Replicators
+     */
+    start() {
+        logger.info('Starting Replication Engine...');
+        for (const replicator of this.replicators.values()) {
+            logger.info(`Starting Replicator: ${replicator.id}`);
+            replicator.start();
+        }
+        logger.info('Successfully started Replication Engine.');
+    }
+    /**
+     *  Stop replication on all managed Replicators
+     */
+    async shutDown() {
+        logger.info('Shutting down Replication Engine...');
+        for (const replicator of this.replicators.values()) {
+            logger.info(`Stopping Replicator: ${replicator.id}`);
+            await replicator.stop();
+        }
+        logger.info('Successfully shut down Replication Engine.');
+    }
+}
+//# sourceMappingURL=ReplicationEngine.js.map

package/dist/replication/ReplicationEngine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ReplicationEngine.js","sourceRoot":"","sources":["../../src/replication/ReplicationEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAG3D,MAAM,OAAO,iBAAiB;IAA9B;QACmB,gBAAW,GAAoC,IAAI,GAAG,EAAE,CAAC;IAsC5E,CAAC;IApCC;;;;OAIG;IACI,QAAQ,CAAC,UAA8B;QAC5C,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,uBAAuB,UAAU,CAAC,EAAE,qBAAqB,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,sCAAsC,UAAU,CAAC,EAAE,yBAAyB,CAAC,CAAC;QAC1F,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACI,KAAK;QACV,MAAM,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;QAC9C,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC;YACnD,MAAM,CAAC,IAAI,CAAC,wBAAwB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;YACrD,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,QAAQ;QACnB,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;QACnD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC;YACnD,MAAM,CAAC,IAAI,CAAC,wBAAwB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;YACrD,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;IAC5D,CAAC;CACF"}

package/dist/replication/ReplicationModule.d.ts

@@ -0,0 +1,51 @@
+import { DataSourceConfig } from '@powersync/service-types/dist/config/PowerSyncConfig.js';
+import * as t from 'ts-codec';
+import * as api from '../api/api-index.js';
+import * as modules from '../modules/modules-index.js';
+import * as system from '../system/system-index.js';
+import { AbstractReplicator } from './AbstractReplicator.js';
+/**
+ *  Provides a common interface for testing the connection to a DataSource.
+ */
+export interface ConnectionTester<TConfig extends DataSourceConfig> {
+    /**
+     *  Confirm if a connection can be established to the datasource for the provided datasource configuration
+     *  @param config
+     */
+    testConnection(config: TConfig): Promise<void>;
+}
+export interface ReplicationModuleOptions extends modules.AbstractModuleOptions {
+    type: string;
+    configSchema: t.AnyCodec;
+}
+/**
+ *  A replication module describes all the functionality that PowerSync requires to
+ *  replicate data from a DataSource. Whenever a new data source is added to powersync this class should be extended.
+ */
+export declare abstract class ReplicationModule<TConfig extends DataSourceConfig> extends modules.AbstractModule implements ConnectionTester<TConfig> {
+    protected type: string;
+    protected configSchema: t.AnyCodec;
+    protected decodedConfig: TConfig | undefined;
+    /**
+     * @protected
+     * @param options
+     */
+    protected constructor(options: ReplicationModuleOptions);
+    /**
+     *  Create the RouteAPI adapter for the DataSource required to service the sync API
+     *  endpoints.
+     */
+    protected abstract createRouteAPIAdapter(): api.RouteAPI;
+    /**
+     *  Create the Replicator to be used by the ReplicationEngine.
+     */
+    protected abstract createReplicator(context: system.ServiceContext): AbstractReplicator;
+    abstract testConnection(config: TConfig): Promise<void>;
+    /**
+     *  Register this module's Replicators and RouteAPI adapters if the required configuration is present.
+     */
+    initialize(context: system.ServiceContext): Promise<void>;
+    protected decodeConfig(config: TConfig): void;
+    private validateConfig;
+    protected getDefaultId(dataSourceName: string): string;
+}

package/dist/replication/ReplicationModule.js

@@ -0,0 +1,68 @@
+import * as t from 'ts-codec';
+import * as types from '@powersync/service-types';
+import * as modules from '../modules/modules-index.js';
+import { schema } from '@powersync/lib-services-framework';
+/**
+ *  A replication module describes all the functionality that PowerSync requires to
+ *  replicate data from a DataSource. Whenever a new data source is added to powersync this class should be extended.
+ */
+export class ReplicationModule extends modules.AbstractModule {
+    /**
+     * @protected
+     * @param options
+     */
+    constructor(options) {
+        super(options);
+        this.type = options.type;
+        this.configSchema = options.configSchema;
+    }
+    /**
+     *  Register this module's Replicators and RouteAPI adapters if the required configuration is present.
+     */
+    async initialize(context) {
+        if (!context.configuration.connections) {
+            // No data source configuration found in the config skip for now
+            return;
+        }
+        const matchingConfig = context.configuration.connections.filter((dataSource) => dataSource.type === this.type);
+        if (!matchingConfig.length) {
+            // No configuration for this module was found
+            return;
+        }
+        if (matchingConfig.length > 1) {
+            this.logger.warning(`Multiple data sources of type ${this.type} found in the configuration. Only the first will be used.`);
+        }
+        try {
+            const baseMatchingConfig = matchingConfig[0];
+            // If decoding fails, log the error and continue, no replication will happen for this data source
+            this.decodeConfig(baseMatchingConfig);
+            context.replicationEngine?.register(this.createReplicator(context));
+            context.routerEngine?.registerAPI(this.createRouteAPIAdapter());
+        }
+        catch (e) {
+            this.logger.error('Failed to initialize.', e);
+        }
+    }
+    decodeConfig(config) {
+        this.validateConfig(config);
+        this.decodedConfig = this.configSchema.decode(config);
+    }
+    validateConfig(config) {
+        const validator = schema
+            .parseJSONSchema(
+        // This generates a schema for the encoded form of the codec
+        t.generateJSONSchema(this.configSchema, {
+            allowAdditional: true,
+            parsers: [types.configFile.portParser]
+        }))
+            .validator();
+        const valid = validator.validate(config);
+        if (!valid.valid) {
+            throw new Error(`Failed to validate Module ${this.name} configuration: ${valid.errors.join(', ')}`);
+        }
+    }
+    getDefaultId(dataSourceName) {
+        return `${this.type}-${dataSourceName}`;
+    }
+}
+//# sourceMappingURL=ReplicationModule.js.map

package/dist/replication/ReplicationModule.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ReplicationModule.js","sourceRoot":"","sources":["../../src/replication/ReplicationModule.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,OAAO,KAAK,KAAK,MAAM,0BAA0B,CAAC;AAElD,OAAO,KAAK,OAAO,MAAM,6BAA6B,CAAC;AAEvD,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAoB3D;;;GAGG;AACH,MAAM,OAAgB,iBACpB,SAAQ,OAAO,CAAC,cAAc;IAO9B;;;OAGG;IACH,YAAsB,OAAiC;QACrD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAC3C,CAAC;IAeD;;OAEG;IACI,KAAK,CAAC,UAAU,CAAC,OAA8B;QACpD,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;YACvC,gEAAgE;YAChE,OAAO;QACT,CAAC;QAED,MAAM,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/G,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC;YAC3B,6CAA6C;YAC7C,OAAO;QACT,CAAC;QAED,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,iCAAiC,IAAI,CAAC,IAAI,2DAA2D,CACtG,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,kBAAkB,GAAG,cAAc,CAAC,CAAC,CAAY,CAAC;YACxD,iGAAiG;YACjG,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC;YAEtC,OAAO,CAAC,iBAAiB,EAAE,QAAQ,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;YACpE,OAAO,CAAC,YAAY,EAAE,WAAW,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC,CAAC;QAClE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAES,YAAY,CAAC,MAAe;QACpC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACxD,CAAC;IAEO,cAAc,CAAC,MAAe;QACpC,MAAM,SAAS,GAAG,MAAM;aACrB,eAAe;QACd,4DAA4D;QAC5D,CAAC,CAAC,kBAAkB,CAAC,IAAI,CAAC,YAAY,EAAE;YACtC,eAAe,EAAE,IAAI;YACrB,OAAO,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC;SACvC,CAAC,CACH;aACA,SAAS,EAAE,CAAC;QAEf,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEzC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,CAAC,IAAI,mBAAmB,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtG,CAAC;IACH,CAAC;IAES,YAAY,CAAC,cAAsB;QAC3C,OAAO,GAAG,IAAI,CAAC,IAAI,IAAI,cAAc,EAAE,CAAC;IAC1C,CAAC;CACF"}

package/dist/replication/replication-index.d.ts

@@ -1,7 +1,5 @@
+export * from './AbstractReplicationJob.js';
+export * from './AbstractReplicator.js';
 export * from './ErrorRateLimiter.js';
-export * from './PgRelation.js';
-export * from './util.js';
-export * from './WalConnection.js';
-export * from './WalStream.js';
-export * from './WalStreamManager.js';
-export * from './WalStreamRunner.js';
+export * from './ReplicationEngine.js';
+export * from './ReplicationModule.js';

package/dist/replication/replication-index.js

@@ -1,8 +1,6 @@
+export * from './AbstractReplicationJob.js';
+export * from './AbstractReplicator.js';
 export * from './ErrorRateLimiter.js';
-export * from './PgRelation.js';
-export * from './util.js';
-export * from './WalConnection.js';
-export * from './WalStream.js';
-export * from './WalStreamManager.js';
-export * from './WalStreamRunner.js';
+export * from './ReplicationEngine.js';
+export * from './ReplicationModule.js';
 //# sourceMappingURL=replication-index.js.map

package/dist/replication/replication-index.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-index.js","sourceRoot":"","sources":["../../src/replication/replication-index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC;AACtC,cAAc,iBAAiB,CAAC;AAChC,cAAc,WAAW,CAAC;AAC1B,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC"}
+{"version":3,"file":"replication-index.js","sourceRoot":"","sources":["../../src/replication/replication-index.ts"],"names":[],"mappings":"AAAA,cAAc,6BAA6B,CAAC;AAC5C,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC"}

package/dist/routes/RouterEngine.d.ts

@@ -0,0 +1,42 @@
+import * as api from '../api/api-index.js';
+import { SocketRouteGenerator } from './router-socket.js';
+import { RouteDefinition } from './router.js';
+export type RouterSetupResponse = {
+    onShutdown: () => Promise<void>;
+};
+export type RouterEngineRoutes = {
+    api_routes: RouteDefinition[];
+    stream_routes: RouteDefinition[];
+    socket_routes: SocketRouteGenerator[];
+};
+export type RouterSetup = (routes: RouterEngineRoutes) => Promise<RouterSetupResponse>;
+/**
+ *  Serves as a registry from which SyncAPIs can be retrieved based on Replication DataSource type
+ *  Initially only one SyncAPI per DataSource type is supported
+ */
+export declare class RouterEngine {
+    closed: boolean;
+    routes: RouterEngineRoutes;
+    protected stopHandlers: Set<() => void>;
+    /**
+     * A final cleanup handler to be executed after all stopHandlers
+     */
+    protected cleanupHandler: (() => Promise<void>) | null;
+    private api;
+    constructor();
+    registerAPI(api: api.RouteAPI): void;
+    getAPI(): api.RouteAPI;
+    /**
+     * Starts the router given the configuration provided
+     */
+    start(setup: RouterSetup): Promise<void>;
+    /**
+     * Runs all stop handlers then final cleanup.
+     */
+    shutDown(): Promise<void>;
+    /**
+     * Add a stop handler callback to be executed when the router engine is being
+     * shutdown.
+     */
+    addStopHandler(handler: () => void): () => void;
+}