@powersync/service-core 0.0.2

package/dist/replication/WalStreamManager.js

@@ -0,0 +1,199 @@
+import * as micro from '@journeyapps-platform/micro';
+import { hrtime } from 'node:process';
+import * as util from '../util/util-index.js';
+import { DefaultErrorRateLimiter } from './ErrorRateLimiter.js';
+import { touch } from './WalStream.js';
+import { WalStreamRunner } from './WalStreamRunner.js';
+// 5 minutes
+const PING_INTERVAL = 1000000000n * 300n;
+export class WalStreamManager {
+    constructor(system) {
+        this.streams = new Map();
+        this.stopped = false;
+        // First ping is only after 5 minutes, not when starting
+        this.lastPing = hrtime.bigint();
+        /**
+         * This limits the effect of retries when there is a persistent issue.
+         */
+        this.rateLimiter = new DefaultErrorRateLimiter();
+        this.system = system;
+        this.storage = system.storage;
+    }
+    start() {
+        this.runLoop().catch((e) => {
+            micro.logger.error(`Fatal WalStream error`, e);
+            util.captureException(e);
+            setTimeout(() => {
+                process.exit(1);
+            }, 1000);
+        });
+    }
+    async stop() {
+        this.stopped = true;
+        let promises = [];
+        for (let stream of this.streams.values()) {
+            promises.push(stream.stop());
+        }
+        await Promise.all(promises);
+    }
+    async runLoop() {
+        const configured_sync_rules = await util.loadSyncRules(this.system.config);
+        let configured_lock = undefined;
+        if (configured_sync_rules != null) {
+            micro.logger.info('Loading sync rules from configuration');
+            try {
+                // Configure new sync rules, if it has changed.
+                // In that case, also immediately take out a lock, so that another process doesn't start replication on it.
+                const { updated, persisted_sync_rules, lock } = await this.storage.configureSyncRules(configured_sync_rules, {
+                    lock: true
+                });
+                if (lock) {
+                    configured_lock = lock;
+                }
+            }
+            catch (e) {
+                // Log, but continue with previous sync rules
+                micro.logger.error(`Failed to load sync rules from configuration`, e);
+            }
+        }
+        else {
+            micro.logger.info('No sync rules configured - configure via API');
+        }
+        while (!this.stopped) {
+            await touch();
+            try {
+                const pool = this.system.pgwire_pool;
+                if (pool) {
+                    await this.refresh({ configured_lock });
+                    // The lock is only valid on the first refresh.
+                    configured_lock = undefined;
+                    // TODO: Ping on all connections when we have multiple
+                    // Perhaps WalStreamRunner would be a better place to do pings?
+                    // We don't ping while in error retry back-off, to avoid having too
+                    // many authentication failures.
+                    if (this.rateLimiter.mayPing()) {
+                        await this.ping(pool);
+                    }
+                }
+            }
+            catch (e) {
+                micro.logger.error(`Failed to refresh wal streams`, e);
+            }
+            await new Promise((resolve) => setTimeout(resolve, 5000));
+        }
+    }
+    /**
+     * Postgres on RDS writes performs a WAL checkpoint every 5 minutes by default, which creates a new 64MB file.
+     *
+     * The old WAL files are only deleted once no replication slot still references it.
+     *
+     * Unfortunately, when there are no changes to the db, the database creates new WAL files without the replication slot
+     * advancing**.
+     *
+     * As a workaround, we write a new message every couple of minutes, to make sure that the replication slot advances.
+     *
+     * **This may be a bug in pgwire or how we're using it.
+     */
+    async ping(db) {
+        const now = hrtime.bigint();
+        if (now - this.lastPing >= PING_INTERVAL) {
+            try {
+                await db.query(`SELECT * FROM pg_logical_emit_message(false, 'powersync', 'ping')`);
+            }
+            catch (e) {
+                micro.logger.warn(`Failed to ping`, e);
+            }
+            this.lastPing = now;
+        }
+    }
+    async refresh(options) {
+        if (this.stopped) {
+            return;
+        }
+        let configured_lock = options?.configured_lock;
+        const existingStreams = new Map(this.streams.entries());
+        const replicating = await this.storage.getReplicatingSyncRules();
+        const newStreams = new Map();
+        for (let syncRules of replicating) {
+            const existing = existingStreams.get(syncRules.id);
+            if (existing && !existing.stopped) {
+                // No change
+                existingStreams.delete(syncRules.id);
+                newStreams.set(syncRules.id, existing);
+            }
+            else if (existing && existing.stopped) {
+                // Stopped (e.g. fatal error, slot rename).
+                // Remove from the list. Next refresh call will restart the stream.
+                existingStreams.delete(syncRules.id);
+            }
+            else {
+                // New (or resume after restart)
+                try {
+                    let lock;
+                    if (configured_lock?.sync_rules_id == syncRules.id) {
+                        lock = configured_lock;
+                    }
+                    else {
+                        lock = await syncRules.lock();
+                    }
+                    const parsed = syncRules.parsed();
+                    const storage = this.storage.getInstance(parsed);
+                    const stream = new WalStreamRunner({
+                        factory: this.storage,
+                        storage: storage,
+                        source_db: this.system.config.connection,
+                        lock,
+                        rateLimiter: this.rateLimiter
+                    });
+                    newStreams.set(syncRules.id, stream);
+                    stream.start();
+                }
+                catch (e) {
+                    // Could be a sync rules parse error,
+                    // for example from stricter validation that was added.
+                    // This will be retried every couple of seconds.
+                    // When new (valid) sync rules are deployed and processed, this one be disabled.
+                    micro.logger.error(`Failed to start replication for ${syncRules.slot_name}`, e);
+                }
+            }
+        }
+        this.streams = newStreams;
+        // TODO: Should this termination be happening in the "background" instead?
+        // That becomes tricky to manage
+        for (let stream of existingStreams.values()) {
+            // Old - stop and remove.
+            try {
+                await stream.terminate();
+            }
+            catch (e) {
+                // This will be retried
+                micro.logger.warn(`Failed to terminate ${stream.slot_name}`, e);
+            }
+        }
+        // Sync rules stopped previously or by a different process.
+        const stopped = await this.storage.getStoppedSyncRules();
+        for (let syncRules of stopped) {
+            try {
+                const lock = await syncRules.lock();
+                try {
+                    const parsed = syncRules.parsed();
+                    const storage = this.storage.getInstance(parsed);
+                    const stream = new WalStreamRunner({
+                        factory: this.storage,
+                        storage: storage,
+                        source_db: this.system.config.connection,
+                        lock
+                    });
+                    await stream.terminate();
+                }
+                finally {
+                    await lock.release();
+                }
+            }
+            catch (e) {
+                micro.logger.warn(`Failed to terminate ${syncRules.slot_name}`, e);
+            }
+        }
+    }
+}
+//# sourceMappingURL=WalStreamManager.js.map

package/dist/replication/WalStreamManager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"WalStreamManager.js","sourceRoot":"","sources":["../../src/replication/WalStreamManager.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,KAAK,MAAM,6BAA6B,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAGtC,OAAO,KAAK,IAAI,MAAM,sBAAsB,CAAC;AAE7C,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAGvD,YAAY;AACZ,MAAM,aAAa,GAAG,WAAc,GAAG,IAAI,CAAC;AAE5C,MAAM,OAAO,gBAAgB;IAiB3B,YAAY,MAA2B;QAhB/B,YAAO,GAAG,IAAI,GAAG,EAA2B,CAAC;QAI7C,YAAO,GAAG,KAAK,CAAC;QAExB,wDAAwD;QAChD,aAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QAInC;;WAEG;QACK,gBAAW,GAAG,IAAI,uBAAuB,EAAE,CAAC;QAGlD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAChC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACzB,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC;YAC/C,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YACzB,UAAU,CAAC,GAAG,EAAE;gBACd,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,EAAE,IAAI,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,QAAQ,GAAoB,EAAE,CAAC;QACnC,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YACzC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/B,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3E,IAAI,eAAe,GAAwC,SAAS,CAAC;QACrE,IAAI,qBAAqB,IAAI,IAAI,EAAE,CAAC;YAClC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;YAC3D,IAAI,CAAC;gBACH,+CAA+C;gBAC/C,2GAA2G;gBAC3G,MAAM,EAAE,OAAO,EAAE,oBAAoB,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,qBAAsB,EAAE;oBAC5G,IAAI,EAAE,IAAI;iBACX,CAAC,CAAC;gBACH,IAAI,IAAI,EAAE,CAAC;oBACT,eAAe,GAAG,IAAI,CAAC;gBACzB,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,6CAA6C;gBAC7C,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,8CAA8C,EAAE,CAAC,CAAC,CAAC;YACxE,CAAC;QACH,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;QACpE,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,KAAK,EAAE,CAAC;YACd,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;gBACrC,IAAI,IAAI,EAAE,CAAC;oBACT,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC;oBACxC,+CAA+C;oBAC/C,eAAe,GAAG,SAAS,CAAC;oBAE5B,sDAAsD;oBACtD,+DAA+D;oBAC/D,mEAAmE;oBACnE,gCAAgC;oBAChC,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;wBAC/B,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACxB,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC,CAAC,CAAC;YACzD,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAED;;;;;;;;;;;OAWG;IACK,KAAK,CAAC,IAAI,CAAC,EAAmB;QACpC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QAC5B,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,IAAI,aAAa,EAAE,CAAC;YACzC,IAAI,CAAC;gBACH,MAAM,EAAE,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC;YACtF,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;YACzC,CAAC;YACD,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;QACtB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,OAAuD;QAC3E,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,IAAI,eAAe,GAAG,OAAO,EAAE,eAAe,CAAC;QAE/C,MAAM,eAAe,GAAG,IAAI,GAAG,CAA0B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACjF,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,CAAC;QACjE,MAAM,UAAU,GAAG,IAAI,GAAG,EAA2B,CAAC;QACtD,KAAK,IAAI,SAAS,IAAI,WAAW,EAAE,CAAC;YAClC,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACnD,IAAI,QAAQ,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;gBAClC,YAAY;gBACZ,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBACrC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;YACzC,CAAC;iBAAM,IAAI,QAAQ,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACxC,2CAA2C;gBAC3C,mEAAmE;gBACnE,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,gCAAgC;gBAChC,IAAI,CAAC;oBACH,IAAI,IAA6B,CAAC;oBAClC,IAAI,eAAe,EAAE,aAAa,IAAI,SAAS,CAAC,EAAE,EAAE,CAAC;wBACnD,IAAI,GAAG,eAAe,CAAC;oBACzB,CAAC;yBAAM,CAAC;wBACN,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;oBAChC,CAAC;oBACD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;oBAClC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;oBACjD,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;wBACjC,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,OAAO,EAAE,OAAO;wBAChB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,UAAW;wBACzC,IAAI;wBACJ,WAAW,EAAE,IAAI,CAAC,WAAW;qBAC9B,CAAC,CAAC;oBACH,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;oBACrC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,qCAAqC;oBACrC,uDAAuD;oBACvD,gDAAgD;oBAChD,gFAAgF;oBAChF,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,SAAS,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;gBAClF,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,UAAU,CAAC;QAE1B,0EAA0E;QAC1E,gCAAgC;QAEhC,KAAK,IAAI,MAAM,IAAI,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;YAC5C,yBAAyB;YACzB,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,uBAAuB;gBACvB,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;QAED,2DAA2D;QAC3D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;QACzD,KAAK,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBACpC,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;oBAClC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;oBACjD,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;wBACjC,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,OAAO,EAAE,OAAO;wBAChB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,UAAW;wBACzC,IAAI;qBACL,CAAC,CAAC;oBACH,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;gBAC3B,CAAC;wBAAS,CAAC;oBACT,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;gBACvB,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,SAAS,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;YACrE,CAAC;QACH,CAAC;IACH,CAAC;CACF"}

package/dist/replication/WalStreamRunner.d.ts

@@ -0,0 +1,38 @@
+import * as storage from '../storage/storage-index.js';
+import { ErrorRateLimiter } from './ErrorRateLimiter.js';
+import { ResolvedConnection } from '../util/config/types.js';
+export interface WalStreamRunnerOptions {
+    factory: storage.BucketStorageFactory;
+    storage: storage.SyncRulesBucketStorage;
+    source_db: ResolvedConnection;
+    lock: storage.ReplicationLock;
+    rateLimiter?: ErrorRateLimiter;
+}
+export declare class WalStreamRunner {
+    options: WalStreamRunnerOptions;
+    private abortController;
+    private runPromise?;
+    private connections;
+    private rateLimiter?;
+    constructor(options: WalStreamRunnerOptions);
+    start(): void;
+    get slot_name(): string;
+    get stopped(): boolean;
+    run(): Promise<void>;
+    replicateLoop(): Promise<void>;
+    replicateOnce(): Promise<void>;
+    /**
+     * This will also release the lock if start() was called earlier.
+     */
+    stop(options?: {
+        force?: boolean;
+    }): Promise<void>;
+    /**
+     * Terminate this replication stream. This drops the replication slot and deletes the replication data.
+     *
+     * Stops replication if needed.
+     */
+    terminate(options?: {
+        force?: boolean;
+    }): Promise<void>;
+}

package/dist/replication/WalStreamRunner.js

@@ -0,0 +1,155 @@
+import * as pgwire from '@powersync/service-jpgwire';
+import * as micro from '@journeyapps-platform/micro';
+import * as util from '../util/util-index.js';
+import { MissingReplicationSlotError, WalStream } from './WalStream.js';
+export class WalStreamRunner {
+    constructor(options) {
+        this.options = options;
+        this.abortController = new AbortController();
+        this.connections = null;
+        this.rateLimiter = options.rateLimiter;
+    }
+    start() {
+        this.runPromise = this.run();
+    }
+    get slot_name() {
+        return this.options.storage.slot_name;
+    }
+    get stopped() {
+        return this.abortController.signal.aborted;
+    }
+    async run() {
+        try {
+            await this.replicateLoop();
+        }
+        catch (e) {
+            // Fatal exception
+            util.captureException(e, {
+                metadata: {
+                    replication_slot: this.slot_name
+                }
+            });
+            micro.logger.error(`Replication failed on ${this.slot_name}`, e);
+            if (e instanceof MissingReplicationSlotError) {
+                // This stops replication on this slot, and creates a new slot
+                await this.options.storage.factory.slotRemoved(this.slot_name);
+            }
+        }
+        finally {
+            this.abortController.abort();
+        }
+        await this.options.lock.release();
+    }
+    async replicateLoop() {
+        while (!this.stopped) {
+            await this.replicateOnce();
+            if (!this.stopped) {
+                await new Promise((resolve) => setTimeout(resolve, 5000));
+            }
+        }
+    }
+    async replicateOnce() {
+        // New connections on every iteration (every error with retry),
+        // otherwise we risk repeating errors related to the connection,
+        // such as caused by cached PG schemas.
+        let connections = new util.PgManager(this.options.source_db, {
+            // Pool connections are only used intermittently.
+            idleTimeout: 30000,
+            maxSize: 2
+        });
+        this.connections = connections;
+        try {
+            await this.rateLimiter?.waitUntilAllowed({ signal: this.abortController.signal });
+            if (this.stopped) {
+                return;
+            }
+            const stream = new WalStream({
+                abort_signal: this.abortController.signal,
+                factory: this.options.factory,
+                storage: this.options.storage,
+                connections
+            });
+            await stream.replicate();
+        }
+        catch (e) {
+            micro.logger.error(`Replication error`, e);
+            if (e.cause != null) {
+                // Example:
+                // PgError.conn_ended: Unable to do postgres query on ended connection
+                //     at PgConnection.stream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:315:13)
+                //     at stream.next (<anonymous>)
+                //     at PgResult.fromStream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:1174:22)
+                //     at PgConnection.query (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:311:21)
+                //     at WalStream.startInitialReplication (file:///.../powersync/powersync-service/lib/replication/WalStream.js:266:22)
+                //     ...
+                //   cause: TypeError: match is not iterable
+                //       at timestamptzToSqlite (file:///.../powersync/packages/jpgwire/dist/util.js:140:50)
+                //       at PgType.decode (file:///.../powersync/packages/jpgwire/dist/pgwire_types.js:25:24)
+                //       at PgConnection._recvDataRow (file:///.../powersync/packages/jpgwire/dist/util.js:88:22)
+                //       at PgConnection._recvMessages (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:656:30)
+                //       at PgConnection._ioloopAttempt (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:563:20)
+                //       at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
+                //       at async PgConnection._ioloop (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:517:14),
+                //   [Symbol(pg.ErrorCode)]: 'conn_ended',
+                //   [Symbol(pg.ErrorResponse)]: undefined
+                // }
+                // Without this additional log, the cause would not be visible in the logs.
+                micro.logger.error(`cause`, e.cause);
+            }
+            if (e instanceof MissingReplicationSlotError) {
+                throw e;
+            }
+            else {
+                // Report the error if relevant, before retrying
+                util.captureException(e, {
+                    metadata: {
+                        replication_slot: this.slot_name
+                    }
+                });
+                // This sets the retry delay
+                this.rateLimiter?.reportError(e);
+            }
+        }
+        finally {
+            this.connections = null;
+            if (connections != null) {
+                await connections.end();
+            }
+        }
+    }
+    /**
+     * This will also release the lock if start() was called earlier.
+     */
+    async stop(options) {
+        micro.logger.info(`${this.slot_name} Stopping replication`);
+        // End gracefully
+        this.abortController.abort();
+        if (options?.force) {
+            // destroy() is more forceful.
+            await this.connections?.destroy();
+        }
+        await this.runPromise;
+    }
+    /**
+     * Terminate this replication stream. This drops the replication slot and deletes the replication data.
+     *
+     * Stops replication if needed.
+     */
+    async terminate(options) {
+        micro.logger.info(`${this.slot_name} Terminating replication`);
+        await this.stop(options);
+        const slotName = this.slot_name;
+        const db = await pgwire.connectPgWire(this.options.source_db, { type: 'standard' });
+        try {
+            await db.query({
+                statement: 'SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = $1',
+                params: [{ type: 'varchar', value: slotName }]
+            });
+        }
+        finally {
+            await db.end();
+        }
+        await this.options.storage.terminate();
+    }
+}
+//# sourceMappingURL=WalStreamRunner.js.map

package/dist/replication/WalStreamRunner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"WalStreamRunner.js","sourceRoot":"","sources":["../../src/replication/WalStreamRunner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AACrD,OAAO,KAAK,KAAK,MAAM,6BAA6B,CAAC;AAGrD,OAAO,KAAK,IAAI,MAAM,sBAAsB,CAAC;AAG7C,OAAO,EAAE,2BAA2B,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAWxE,MAAM,OAAO,eAAe;IAS1B,YAAmB,OAA+B;QAA/B,YAAO,GAAP,OAAO,CAAwB;QAR1C,oBAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QAIxC,gBAAW,GAA0B,IAAI,CAAC;QAKhD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IACzC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC/B,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC;IACxC,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,GAAG;QACP,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,kBAAkB;YAClB,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE;gBACvB,QAAQ,EAAE;oBACR,gBAAgB,EAAE,IAAI,CAAC,SAAS;iBACjC;aACF,CAAC,CAAC;YACH,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;YAEjE,IAAI,CAAC,YAAY,2BAA2B,EAAE,CAAC;gBAC7C,8DAA8D;gBAC9D,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACjE,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;QACD,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAE3B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAClB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,+DAA+D;QAC/D,gEAAgE;QAChE,uCAAuC;QACvC,IAAI,WAAW,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAC3D,iDAAiD;YACjD,WAAW,EAAE,KAAM;YACnB,OAAO,EAAE,CAAC;SACX,CAAC,CAAC;QACH,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC;YAClF,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;gBAC3B,YAAY,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;gBACzC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;gBAC7B,WAAW;aACZ,CAAC,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;QAC3B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC;YAC3C,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;gBACpB,WAAW;gBACX,sEAAsE;gBACtE,2KAA2K;gBAC3K,mCAAmC;gBACnC,4KAA4K;gBAC5K,0KAA0K;gBAC1K,yHAAyH;gBACzH,UAAU;gBACV,4CAA4C;gBAC5C,4FAA4F;gBAC5F,6FAA6F;gBAC7F,iGAAiG;gBACjG,oLAAoL;gBACpL,qLAAqL;gBACrL,sFAAsF;gBACtF,qLAAqL;gBACrL,0CAA0C;gBAC1C,0CAA0C;gBAC1C,IAAI;gBACJ,2EAA2E;gBAC3E,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YACvC,CAAC;YACD,IAAI,CAAC,YAAY,2BAA2B,EAAE,CAAC;gBAC7C,MAAM,CAAC,CAAC;YACV,CAAC;iBAAM,CAAC;gBACN,gDAAgD;gBAChD,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE;oBACvB,QAAQ,EAAE;wBACR,gBAAgB,EAAE,IAAI,CAAC,SAAS;qBACjC;iBACF,CAAC,CAAC;gBACH,4BAA4B;gBAC5B,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;gBACxB,MAAM,WAAW,CAAC,GAAG,EAAE,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,OAA6B;QACtC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,uBAAuB,CAAC,CAAC;QAC5D,iBAAiB;QACjB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAE7B,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC;YACnB,8BAA8B;YAC9B,MAAM,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,CAAC;QACpC,CAAC;QACD,MAAM,IAAI,CAAC,UAAU,CAAC;IACxB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,SAAS,CAAC,OAA6B;QAC3C,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,0BAA0B,CAAC,CAAC;QAC/D,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEzB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QACpF,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,KAAK,CAAC;gBACb,SAAS,EAAE,2FAA2F;gBACtG,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;aAC/C,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC;QACjB,CAAC;QAED,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;IACzC,CAAC;CACF"}

package/dist/replication/replication-index.d.ts

@@ -0,0 +1,7 @@
+export * from './ErrorRateLimiter.js';
+export * from './PgRelation.js';
+export * from './util.js';
+export * from './WalConnection.js';
+export * from './WalStream.js';
+export * from './WalStreamManager.js';
+export * from './WalStreamRunner.js';

package/dist/replication/replication-index.js

@@ -0,0 +1,8 @@
+export * from './ErrorRateLimiter.js';
+export * from './PgRelation.js';
+export * from './util.js';
+export * from './WalConnection.js';
+export * from './WalStream.js';
+export * from './WalStreamManager.js';
+export * from './WalStreamRunner.js';
+//# sourceMappingURL=replication-index.js.map

package/dist/replication/replication-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"replication-index.js","sourceRoot":"","sources":["../../src/replication/replication-index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC;AACtC,cAAc,iBAAiB,CAAC;AAChC,cAAc,WAAW,CAAC;AAC1B,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC"}

package/dist/replication/util.d.ts

@@ -0,0 +1,9 @@
+import * as pgwire from '@powersync/service-jpgwire';
+import { ReplicationColumn, ReplicationIdentity } from './PgRelation.js';
+export interface ReplicaIdentityResult {
+    columns: ReplicationColumn[];
+    replicationIdentity: ReplicationIdentity;
+}
+export declare function getPrimaryKeyColumns(db: pgwire.PgClient, relationId: number, mode: 'primary' | 'replident'): Promise<ReplicationColumn[]>;
+export declare function getAllColumns(db: pgwire.PgClient, relationId: number): Promise<ReplicationColumn[]>;
+export declare function getReplicationIdentityColumns(db: pgwire.PgClient, relationId: number): Promise<ReplicaIdentityResult>;

package/dist/replication/util.js

@@ -0,0 +1,62 @@
+import * as util from '../util/util-index.js';
+export async function getPrimaryKeyColumns(db, relationId, mode) {
+    const indexFlag = mode == 'primary' ? `i.indisprimary` : `i.indisreplident`;
+    const attrRows = await util.retriedQuery(db, {
+        statement: `SELECT a.attname as name, a.atttypid as typeid, a.attnum as attnum
+                                    FROM pg_index i
+                                             JOIN pg_attribute a
+                                                  ON a.attrelid = i.indrelid
+                                                      AND a.attnum = ANY (i.indkey)
+                                    WHERE i.indrelid = $1::oid
+                                      AND ${indexFlag}
+                                      AND a.attnum > 0
+                                      ORDER BY a.attnum`,
+        params: [{ value: relationId, type: 'int4' }]
+    });
+    return attrRows.rows.map((row) => {
+        return { name: row[0], typeOid: row[1] };
+    });
+}
+export async function getAllColumns(db, relationId) {
+    const attrRows = await util.retriedQuery(db, {
+        statement: `SELECT a.attname as name, a.atttypid as typeid, a.attnum as attnum
+                                    FROM pg_attribute a
+                                    WHERE a.attrelid = $1::oid
+                                      AND attnum > 0
+                                    ORDER BY a.attnum`,
+        params: [{ type: 'varchar', value: relationId }]
+    });
+    return attrRows.rows.map((row) => {
+        return { name: row[0], typeOid: row[1] };
+    });
+}
+export async function getReplicationIdentityColumns(db, relationId) {
+    const rows = await util.retriedQuery(db, {
+        statement: `SELECT CASE relreplident
+        WHEN 'd' THEN 'default'
+        WHEN 'n' THEN 'nothing'
+        WHEN 'f' THEN 'full'
+        WHEN 'i' THEN 'index'
+     END AS replica_identity
+FROM pg_class
+WHERE oid = $1::oid LIMIT 1`,
+        params: [{ type: 'int8', value: relationId }]
+    });
+    const idType = rows.rows[0]?.[0];
+    if (idType == 'nothing' || idType == null) {
+        return { replicationIdentity: 'nothing', columns: [] };
+    }
+    else if (idType == 'full') {
+        return { replicationIdentity: 'full', columns: await getAllColumns(db, relationId) };
+    }
+    else if (idType == 'default') {
+        return { replicationIdentity: 'default', columns: await getPrimaryKeyColumns(db, relationId, 'primary') };
+    }
+    else if (idType == 'index') {
+        return { replicationIdentity: 'index', columns: await getPrimaryKeyColumns(db, relationId, 'replident') };
+    }
+    else {
+        return { replicationIdentity: 'nothing', columns: [] };
+    }
+}
+//# sourceMappingURL=util.js.map

package/dist/replication/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/replication/util.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,IAAI,MAAM,sBAAsB,CAAC;AAQ7C,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,EAAmB,EACnB,UAAkB,EAClB,IAA6B;IAE7B,MAAM,SAAS,GAAG,IAAI,IAAI,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,kBAAkB,CAAC;IAC5E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE;QAC3C,SAAS,EAAE;;;;;;4CAM6B,SAAS;;wDAEG;QACpD,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;KAC9C,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC/B,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAW,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAW,EAAE,CAAC;IAC/D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,EAAmB,EAAE,UAAkB;IACzE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE;QAC3C,SAAS,EAAE;;;;sDAIuC;QAClD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;KACjD,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC/B,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAW,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAW,EAAE,CAAC;IAC/D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,EAAmB,EACnB,UAAkB;IAElB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE;QACvC,SAAS,EAAE;;;;;;;4BAOa;QACxB,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;KAC9C,CAAC,CAAC;IACH,MAAM,MAAM,GAAW,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACzC,IAAI,MAAM,IAAI,SAAS,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QAC1C,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACzD,CAAC;SAAM,IAAI,MAAM,IAAI,MAAM,EAAE,CAAC;QAC5B,OAAO,EAAE,mBAAmB,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC,EAAE,EAAE,UAAU,CAAC,EAAE,CAAC;IACvF,CAAC;SAAM,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;QAC/B,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC,EAAE,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,CAAC;IAC5G,CAAC;SAAM,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC,EAAE,EAAE,UAAU,EAAE,WAAW,CAAC,EAAE,CAAC;IAC5G,CAAC;SAAM,CAAC;QACN,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACzD,CAAC;AACH,CAAC"}

package/dist/routes/admin.d.ts

@@ -0,0 +1,7 @@
+import { RouteGenerator } from './router.js';
+export declare const executeSql: RouteGenerator;
+export declare const diagnostics: RouteGenerator;
+export declare const getSchema: RouteGenerator;
+export declare const reprocess: RouteGenerator;
+export declare const validate: RouteGenerator;
+export declare const admin_routes: RouteGenerator[];