@powersync/service-module-postgres 0.0.0-dev-20240918092408

package/dist/replication/ConnectionManagerFactory.js

@@ -0,0 +1,21 @@
+import { PgManager } from './PgManager.js';
+import { logger } from '@powersync/lib-services-framework';
+export class ConnectionManagerFactory {
+    constructor(dbConnectionConfig) {
+        this.dbConnectionConfig = dbConnectionConfig;
+        this.connectionManagers = [];
+    }
+    create(poolOptions) {
+        const manager = new PgManager(this.dbConnectionConfig, poolOptions);
+        this.connectionManagers.push(manager);
+        return manager;
+    }
+    async shutdown() {
+        logger.info('Shutting down Postgres connection Managers...');
+        for (const manager of this.connectionManagers) {
+            await manager.end();
+        }
+        logger.info('Postgres connection Managers shutdown completed.');
+    }
+}
+//# sourceMappingURL=ConnectionManagerFactory.js.map

package/dist/replication/ConnectionManagerFactory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ConnectionManagerFactory.js","sourceRoot":"","sources":["../../src/replication/ConnectionManagerFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG3C,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,MAAM,OAAO,wBAAwB;IAInC,YAAY,kBAAsD;QAChE,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC;IAC/B,CAAC;IAED,MAAM,CAAC,WAA0B;QAC/B,MAAM,OAAO,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QACpE,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;QAC7D,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,kBAAkB,EAAE;YAC7C,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC;SACrB;QACD,MAAM,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;IAClE,CAAC;CACF"}

package/dist/replication/PgManager.d.ts

@@ -0,0 +1,25 @@
+import * as pgwire from '@powersync/service-jpgwire';
+import { NormalizedPostgresConnectionConfig } from '../types/types.js';
+export declare class PgManager {
+    options: NormalizedPostgresConnectionConfig;
+    poolOptions: pgwire.PgPoolOptions;
+    /**
+     * Do not use this for any transactions.
+     */
+    readonly pool: pgwire.PgClient;
+    private connectionPromises;
+    constructor(options: NormalizedPostgresConnectionConfig, poolOptions: pgwire.PgPoolOptions);
+    get connectionTag(): string;
+    /**
+     * Create a new replication connection.
+     */
+    replicationConnection(): Promise<pgwire.PgConnection>;
+    /**
+     * Create a new standard connection, used for initial snapshot.
+     *
+     * This connection must not be shared between multiple async contexts.
+     */
+    snapshotConnection(): Promise<pgwire.PgConnection>;
+    end(): Promise<void>;
+    destroy(): Promise<void>;
+}

package/dist/replication/PgManager.js

@@ -0,0 +1,60 @@
+import * as pgwire from '@powersync/service-jpgwire';
+export class PgManager {
+    constructor(options, poolOptions) {
+        this.options = options;
+        this.poolOptions = poolOptions;
+        this.connectionPromises = [];
+        // The pool is lazy - no connections are opened until a query is performed.
+        this.pool = pgwire.connectPgWirePool(this.options, poolOptions);
+    }
+    get connectionTag() {
+        return this.options.tag;
+    }
+    /**
+     * Create a new replication connection.
+     */
+    async replicationConnection() {
+        const p = pgwire.connectPgWire(this.options, { type: 'replication' });
+        this.connectionPromises.push(p);
+        return await p;
+    }
+    /**
+     * Create a new standard connection, used for initial snapshot.
+     *
+     * This connection must not be shared between multiple async contexts.
+     */
+    async snapshotConnection() {
+        const p = pgwire.connectPgWire(this.options, { type: 'standard' });
+        this.connectionPromises.push(p);
+        return await p;
+    }
+    async end() {
+        for (let result of await Promise.allSettled([
+            this.pool.end(),
+            ...this.connectionPromises.map(async (promise) => {
+                const connection = await promise;
+                return await connection.end();
+            })
+        ])) {
+            // Throw the first error, if any
+            if (result.status == 'rejected') {
+                throw result.reason;
+            }
+        }
+    }
+    async destroy() {
+        this.pool.destroy();
+        for (let result of await Promise.allSettled([
+            ...this.connectionPromises.map(async (promise) => {
+                const connection = await promise;
+                return connection.destroy();
+            })
+        ])) {
+            // Throw the first error, if any
+            if (result.status == 'rejected') {
+                throw result.reason;
+            }
+        }
+    }
+}
+//# sourceMappingURL=PgManager.js.map

package/dist/replication/PgManager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PgManager.js","sourceRoot":"","sources":["../../src/replication/PgManager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAC;AAGrD,MAAM,OAAO,SAAS;IAQpB,YAAmB,OAA2C,EAAS,WAAiC;QAArF,YAAO,GAAP,OAAO,CAAoC;QAAS,gBAAW,GAAX,WAAW,CAAsB;QAFhG,uBAAkB,GAAmC,EAAE,CAAC;QAG9D,2EAA2E;QAC3E,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAClE,CAAC;IAED,IAAW,aAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB;QACzB,MAAM,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,CAAC;QACtE,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,OAAO,MAAM,CAAC,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,kBAAkB;QACtB,MAAM,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,OAAO,MAAM,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,GAAG;QACP,KAAK,IAAI,MAAM,IAAI,MAAM,OAAO,CAAC,UAAU,CAAC;YAC1C,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACf,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;gBAC/C,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC;gBACjC,OAAO,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC;YAChC,CAAC,CAAC;SACH,CAAC,EAAE;YACF,gCAAgC;YAChC,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,EAAE;gBAC/B,MAAM,MAAM,CAAC,MAAM,CAAC;aACrB;SACF;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACpB,KAAK,IAAI,MAAM,IAAI,MAAM,OAAO,CAAC,UAAU,CAAC;YAC1C,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;gBAC/C,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC;gBACjC,OAAO,UAAU,CAAC,OAAO,EAAE,CAAC;YAC9B,CAAC,CAAC;SACH,CAAC,EAAE;YACF,gCAAgC;YAChC,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,EAAE;gBAC/B,MAAM,MAAM,CAAC,MAAM,CAAC;aACrB;SACF;IACH,CAAC;CACF"}

package/dist/replication/PgRelation.d.ts

@@ -0,0 +1,6 @@
+import { storage } from '@powersync/service-core';
+import { PgoutputRelation } from '@powersync/service-jpgwire';
+export type ReplicationIdentity = 'default' | 'nothing' | 'full' | 'index';
+export declare function getReplicaIdColumns(relation: PgoutputRelation): storage.ColumnDescriptor[];
+export declare function getRelId(source: PgoutputRelation): number;
+export declare function getPgOutputRelation(source: PgoutputRelation): storage.SourceEntityDescriptor;

package/dist/replication/PgRelation.js

@@ -0,0 +1,27 @@
+export function getReplicaIdColumns(relation) {
+    if (relation.replicaIdentity == 'nothing') {
+        return [];
+    }
+    else {
+        return relation.columns
+            .filter((c) => (c.flags & 0b1) != 0)
+            .map((c) => ({ name: c.name, typeId: c.typeOid }));
+    }
+}
+export function getRelId(source) {
+    // Source types are wrong here
+    const relId = source.relationOid;
+    if (!relId) {
+        throw new Error(`No relation id!`);
+    }
+    return relId;
+}
+export function getPgOutputRelation(source) {
+    return {
+        name: source.name,
+        schema: source.schema,
+        objectId: getRelId(source),
+        replicationColumns: getReplicaIdColumns(source)
+    };
+}
+//# sourceMappingURL=PgRelation.js.map

package/dist/replication/PgRelation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PgRelation.js","sourceRoot":"","sources":["../../src/replication/PgRelation.ts"],"names":[],"mappings":"AAKA,MAAM,UAAU,mBAAmB,CAAC,QAA0B;IAC5D,IAAI,QAAQ,CAAC,eAAe,IAAI,SAAS,EAAE;QACzC,OAAO,EAAE,CAAC;KACX;SAAM;QACL,OAAO,QAAQ,CAAC,OAAO;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;aACnC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAqC,CAAC,CAAC,CAAC;KACzF;AACH,CAAC;AACD,MAAM,UAAU,QAAQ,CAAC,MAAwB;IAC/C,8BAA8B;IAC9B,MAAM,KAAK,GAAI,MAAc,CAAC,WAAqB,CAAC;IACpD,IAAI,CAAC,KAAK,EAAE;QACV,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;KACpC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,MAAwB;IAC1D,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC;QAC1B,kBAAkB,EAAE,mBAAmB,CAAC,MAAM,CAAC;KACP,CAAC;AAC7C,CAAC"}

package/dist/replication/PostgresErrorRateLimiter.d.ts

@@ -0,0 +1,11 @@
+/// <reference types="node" resolution-mode="require"/>
+import { ErrorRateLimiter } from '@powersync/service-core';
+export declare class PostgresErrorRateLimiter implements ErrorRateLimiter {
+    nextAllowed: number;
+    waitUntilAllowed(options?: {
+        signal?: AbortSignal | undefined;
+    } | undefined): Promise<void>;
+    mayPing(): boolean;
+    reportError(e: any): void;
+    private setDelay;
+}

package/dist/replication/PostgresErrorRateLimiter.js

@@ -0,0 +1,43 @@
+import { setTimeout } from 'timers/promises';
+export class PostgresErrorRateLimiter {
+    constructor() {
+        this.nextAllowed = Date.now();
+    }
+    async waitUntilAllowed(options) {
+        const delay = Math.max(0, this.nextAllowed - Date.now());
+        // Minimum delay between connections, even without errors
+        this.setDelay(500);
+        await setTimeout(delay, undefined, { signal: options?.signal });
+    }
+    mayPing() {
+        return Date.now() >= this.nextAllowed;
+    }
+    reportError(e) {
+        const message = e.message ?? '';
+        if (message.includes('password authentication failed')) {
+            // Wait 15 minutes, to avoid triggering Supabase's fail2ban
+            this.setDelay(900000);
+        }
+        else if (message.includes('ENOTFOUND')) {
+            // DNS lookup issue - incorrect URI or deleted instance
+            this.setDelay(120000);
+        }
+        else if (message.includes('ECONNREFUSED')) {
+            // Could be fail2ban or similar
+            this.setDelay(120000);
+        }
+        else if (message.includes('Unable to do postgres query on ended pool') ||
+            message.includes('Postgres unexpectedly closed connection')) {
+            // Connection timed out - ignore / immediately retry
+            // We don't explicitly set the delay to 0, since there could have been another error that
+            // we need to respect.
+        }
+        else {
+            this.setDelay(30000);
+        }
+    }
+    setDelay(delay) {
+        this.nextAllowed = Math.max(this.nextAllowed, Date.now() + delay);
+    }
+}
+//# sourceMappingURL=PostgresErrorRateLimiter.js.map

package/dist/replication/PostgresErrorRateLimiter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PostgresErrorRateLimiter.js","sourceRoot":"","sources":["../../src/replication/PostgresErrorRateLimiter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAG7C,MAAM,OAAO,wBAAwB;IAArC;QACE,gBAAW,GAAW,IAAI,CAAC,GAAG,EAAE,CAAC;IAuCnC,CAAC;IArCC,KAAK,CAAC,gBAAgB,CAAC,OAA0D;QAC/E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACzD,yDAAyD;QACzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACnB,MAAM,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC;IACxC,CAAC;IAED,WAAW,CAAC,CAAM;QAChB,MAAM,OAAO,GAAI,CAAC,CAAC,OAAkB,IAAI,EAAE,CAAC;QAC5C,IAAI,OAAO,CAAC,QAAQ,CAAC,gCAAgC,CAAC,EAAE;YACtD,2DAA2D;YAC3D,IAAI,CAAC,QAAQ,CAAC,MAAO,CAAC,CAAC;SACxB;aAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE;YACxC,uDAAuD;YACvD,IAAI,CAAC,QAAQ,CAAC,MAAO,CAAC,CAAC;SACxB;aAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE;YAC3C,+BAA+B;YAC/B,IAAI,CAAC,QAAQ,CAAC,MAAO,CAAC,CAAC;SACxB;aAAM,IACL,OAAO,CAAC,QAAQ,CAAC,2CAA2C,CAAC;YAC7D,OAAO,CAAC,QAAQ,CAAC,yCAAyC,CAAC,EAC3D;YACA,oDAAoD;YACpD,yFAAyF;YACzF,sBAAsB;SACvB;aAAM;YACL,IAAI,CAAC,QAAQ,CAAC,KAAM,CAAC,CAAC;SACvB;IACH,CAAC;IAEO,QAAQ,CAAC,KAAa;QAC5B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;IACpE,CAAC;CACF"}

package/dist/replication/WalStream.d.ts

@@ -0,0 +1,53 @@
+/// <reference types="node" resolution-mode="require"/>
+import * as pgwire from '@powersync/service-jpgwire';
+import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
+import { SourceEntityDescriptor, storage } from '@powersync/service-core';
+import { PgManager } from './PgManager.js';
+export declare const ZERO_LSN = "00000000/00000000";
+export declare const PUBLICATION_NAME = "powersync";
+export declare const POSTGRES_DEFAULT_SCHEMA = "public";
+export interface WalStreamOptions {
+    connections: PgManager;
+    storage: storage.SyncRulesBucketStorage;
+    abort_signal: AbortSignal;
+}
+interface InitResult {
+    needsInitialSync: boolean;
+}
+export declare class MissingReplicationSlotError extends Error {
+    constructor(message: string);
+}
+export declare class WalStream {
+    sync_rules: SqlSyncRules;
+    group_id: number;
+    connection_id: number;
+    private readonly storage;
+    private readonly slot_name;
+    private connections;
+    private abort_signal;
+    private relation_cache;
+    private startedStreaming;
+    constructor(options: WalStreamOptions);
+    get stopped(): boolean;
+    getQualifiedTableNames(batch: storage.BucketStorageBatch, db: pgwire.PgConnection, tablePattern: TablePattern): Promise<storage.SourceTable[]>;
+    initSlot(): Promise<InitResult>;
+    estimatedCount(db: pgwire.PgConnection, table: storage.SourceTable): Promise<string>;
+    /**
+     * Start initial replication.
+     *
+     * If (partial) replication was done before on this slot, this clears the state
+     * and starts again from scratch.
+     */
+    startInitialReplication(replicationConnection: pgwire.PgConnection): Promise<void>;
+    initialReplication(db: pgwire.PgConnection, lsn: string): Promise<void>;
+    static getQueryData(results: Iterable<DatabaseInputRow>): Generator<SqliteRow>;
+    private snapshotTable;
+    handleRelation(batch: storage.BucketStorageBatch, descriptor: SourceEntityDescriptor, snapshot: boolean): Promise<storage.SourceTable>;
+    private getTable;
+    writeChange(batch: storage.BucketStorageBatch, msg: pgwire.PgoutputMessage): Promise<storage.FlushedResult | null>;
+    replicate(): Promise<void>;
+    initReplication(replicationConnection: pgwire.PgConnection): Promise<void>;
+    streamChanges(replicationConnection: pgwire.PgConnection): Promise<void>;
+    ack(lsn: string, replicationStream: pgwire.ReplicationStream): Promise<void>;
+}
+export {};