@rocicorp/zero 0.0.0-202410031711

package/out/zero-cache/src/services/change-streamer/change-streamer.js

@@ -0,0 +1,7 @@
+export var ErrorType;
+(function (ErrorType) {
+    ErrorType[ErrorType["Unknown"] = 0] = "Unknown";
+    ErrorType[ErrorType["WrongReplicaVersion"] = 1] = "WrongReplicaVersion";
+    ErrorType[ErrorType["WatermarkTooOld"] = 2] = "WatermarkTooOld";
+})(ErrorType || (ErrorType = {}));
+//# sourceMappingURL=change-streamer.js.map

package/out/zero-cache/src/services/change-streamer/change-streamer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-streamer.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AA8FA,MAAM,CAAN,IAAkB,SAIjB;AAJD,WAAkB,SAAS;IACzB,+CAAO,CAAA;IACP,uEAAmB,CAAA;IACnB,+DAAe,CAAA;AACjB,CAAC,EAJiB,SAAS,KAAT,SAAS,QAI1B"}

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts

@@ -0,0 +1,19 @@
+import type { WatermarkedChange } from './change-streamer-service.js';
+import { Subscriber } from './subscriber.js';
+export declare class Forwarder {
+    #private;
+    /**
+     * `add()` is called in lock step with `Storer.catchup()` so that the
+     * two components have an equivalent interpretation of whether a Transaction is
+     * currently being streamed.
+     */
+    add(sub: Subscriber): void;
+    remove(sub: Subscriber): void;
+    /**
+     * `forward()` is called in lockstep with `Storer.store()` so that the
+     * two components have an equivalent interpretation of whether a Transaction is
+     * currently being streamed.
+     */
+    forward(entry: WatermarkedChange): void;
+}
+//# sourceMappingURL=forwarder.d.ts.map

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAE3C,qBAAa,SAAS;;IAKpB;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU;IAQnB,MAAM,CAAC,GAAG,EAAE,UAAU;IAMtB;;;;OAIG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB;CA0BjC"}

package/out/zero-cache/src/services/change-streamer/forwarder.js

@@ -0,0 +1,56 @@
+import { Subscriber } from './subscriber.js';
+export class Forwarder {
+    #active = new Set();
+    #queued = new Set();
+    #inTransaction = false;
+    /**
+     * `add()` is called in lock step with `Storer.catchup()` so that the
+     * two components have an equivalent interpretation of whether a Transaction is
+     * currently being streamed.
+     */
+    add(sub) {
+        if (this.#inTransaction) {
+            this.#queued.add(sub);
+        }
+        else {
+            this.#active.add(sub);
+        }
+    }
+    remove(sub) {
+        this.#active.delete(sub);
+        this.#queued.delete(sub);
+        sub.close();
+    }
+    /**
+     * `forward()` is called in lockstep with `Storer.store()` so that the
+     * two components have an equivalent interpretation of whether a Transaction is
+     * currently being streamed.
+     */
+    forward(entry) {
+        const [type] = entry[1];
+        for (const active of this.#active.values()) {
+            active.send(entry);
+        }
+        switch (type) {
+            case 'begin':
+                // While in a Transaction, all added subscribers are "queued" so that no
+                // messages are forwarded to them. This state corresponds to being queued
+                // for catchup in the Storer, which will retrieve historic changes
+                // and call catchup() once the current transaction is committed.
+                this.#inTransaction = true;
+                break;
+            case 'commit':
+                // Upon commit, all queued subscribers are transferred to the active set.
+                // This means that they can receive messages starting from the next transaction.
+                // Note that if catchup is still in progress (in the Storer), these messages
+                // will be buffered in the backlog until catchup completes.
+                this.#inTransaction = false;
+                for (const sub of this.#queued.values()) {
+                    this.#active.add(sub);
+                }
+                this.#queued.clear();
+                break;
+        }
+    }
+}
+//# sourceMappingURL=forwarder.js.map

package/out/zero-cache/src/services/change-streamer/forwarder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"forwarder.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAE3C,MAAM,OAAO,SAAS;IACX,OAAO,GAAG,IAAI,GAAG,EAAc,CAAC;IAChC,OAAO,GAAG,IAAI,GAAG,EAAc,CAAC;IACzC,cAAc,GAAG,KAAK,CAAC;IAEvB;;;;OAIG;IACH,GAAG,CAAC,GAAe;QACjB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,MAAM,CAAC,GAAe;QACpB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,GAAG,CAAC,KAAK,EAAE,CAAC;IACd,CAAC;IAED;;;;OAIG;IACH,OAAO,CAAC,KAAwB;QAC9B,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACxB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QACD,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,OAAO;gBACV,wEAAwE;gBACxE,yEAAyE;gBACzE,kEAAkE;gBAClE,gEAAgE;gBAChE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC3B,MAAM;YACR,KAAK,QAAQ;gBACX,yEAAyE;gBACzE,gFAAgF;gBAChF,4EAA4E;gBAC5E,2DAA2D;gBAC3D,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;gBAC5B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;oBACxC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACxB,CAAC;gBACD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACrB,MAAM;QACV,CAAC;IACH,CAAC;CACF"}

package/out/zero-cache/src/services/change-streamer/pg/change-source.d.ts

@@ -0,0 +1,14 @@
+import { LogContext } from '@rocicorp/logger';
+import type { ChangeSource } from '../change-streamer-service.js';
+import type { ReplicationConfig } from '../schema/tables.js';
+import type { ShardConfig } from './shard-config.js';
+/**
+ * Initializes a Postgres change source, including the initial sync of the
+ * replica, before streaming changes from the corresponding logical replication
+ * stream.
+ */
+export declare function initializeChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string): Promise<{
+    replicationConfig: ReplicationConfig;
+    changeSource: ChangeSource;
+}>;
+//# sourceMappingURL=change-source.d.ts.map

package/out/zero-cache/src/services/change-streamer/pg/change-source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/pg/change-source.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAsB5C,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,+BAA+B,CAAC;AAG9E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,qBAAqB,CAAC;AAI3D,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,mBAAmB,CAAC;AAMnD;;;;GAIG;AACH,wBAAsB,sBAAsB,CAC1C,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAiC7E"}

package/out/zero-cache/src/services/change-streamer/pg/change-source.js

@@ -0,0 +1,206 @@
+import { PG_ADMIN_SHUTDOWN } from '@drdgvhbh/postgres-error-codes';
+import { LogContext } from '@rocicorp/logger';
+import { resolver } from '@rocicorp/resolver';
+import { LogicalReplicationService, Pgoutput, PgoutputPlugin, } from 'pg-logical-replication';
+import { DatabaseError } from 'pg-protocol';
+import { AbortError } from '../../../../../shared/src/abort-error.js';
+import { deepEqual } from '../../../../../shared/src/json.js';
+import { sleep } from '../../../../../shared/src/sleep.js';
+import { StatementRunner } from '../../../../../zero-cache/src/db/statements.js';
+import { stringify } from '../../../../../zero-cache/src/types/bigint-json.js';
+import { max, oneAfter } from '../../../../../zero-cache/src/types/lexi-version.js';
+import { pgClient, registerPostgresTypeParsers, } from '../../../../../zero-cache/src/types/pg.js';
+import { Subscription } from '../../../../../zero-cache/src/types/subscription.js';
+import { Database } from '../../../../../zqlite/src/db.js';
+import { getSubscriptionState } from '../../replicator/schema/replication-state.js';
+import { replicationSlot } from './initial-sync.js';
+import { fromLexiVersion, toLexiVersion } from './lsn.js';
+import { INTERNAL_PUBLICATION_PREFIX } from './schema/zero.js';
+import { initSyncSchema } from './sync-schema.js';
+// BigInt support from LogicalReplicationService.
+registerPostgresTypeParsers();
+/**
+ * Initializes a Postgres change source, including the initial sync of the
+ * replica, before streaming changes from the corresponding logical replication
+ * stream.
+ */
+export async function initializeChangeSource(lc, upstreamURI, shard, replicaDbFile) {
+    await initSyncSchema(lc, 'change-streamer', shard, replicaDbFile, upstreamURI);
+    const replica = new Database(lc, replicaDbFile);
+    const replicationConfig = getSubscriptionState(new StatementRunner(replica));
+    if (shard.publications.length) {
+        // Verify that the publications match what has been synced.
+        const requested = [...shard.publications].sort();
+        const replicated = replicationConfig.publications
+            .filter(p => !p.startsWith(INTERNAL_PUBLICATION_PREFIX))
+            .sort();
+        if (!deepEqual(requested, replicated)) {
+            throw new Error(`Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`);
+        }
+    }
+    const changeSource = new PostgresChangeSource(lc, upstreamURI, shard.id, replicationConfig);
+    return { replicationConfig, changeSource };
+}
+/**
+ * Postgres implementation of a {@link ChangeSource} backed by a logical
+ * replication stream.
+ */
+class PostgresChangeSource {
+    #lc;
+    #upstreamUri;
+    #shardID;
+    #replicationConfig;
+    constructor(lc, upstreamUri, shardID, replicationConfig) {
+        this.#lc = lc.withContext('component', 'change-source');
+        this.#upstreamUri = upstreamUri;
+        this.#shardID = shardID;
+        this.#replicationConfig = replicationConfig;
+    }
+    async startStream(clientWatermark) {
+        const db = pgClient(this.#lc, this.#upstreamUri);
+        const slot = replicationSlot(this.#shardID);
+        const clientStart = oneAfter(clientWatermark);
+        try {
+            let lastLSN = '0/0';
+            const ack = (commit) => {
+                if (commit) {
+                    const { watermark } = commit[2];
+                    lastLSN = fromLexiVersion(watermark);
+                }
+                void service.acknowledge(lastLSN);
+            };
+            const changes = Subscription.create({
+                cleanup: () => service.stop(),
+            });
+            // To avoid a race condition when handing off the replication stream
+            // between tasks, query the `confirmed_flush_lsn` for the replication
+            // slot only after the replication stream starts, as that is when it
+            // is guaranteed not to change (i.e. until we ACK a commit).
+            const { promise: nextWatermark, resolve, reject } = resolver();
+            const service = new LogicalReplicationService({ connectionString: this.#upstreamUri }, { acknowledge: { auto: false, timeoutSeconds: 0 } })
+                .on('start', () => this.getNextWatermark(db, slot, clientStart)
+                .then(resolve)
+                .catch(e => reject(translateError(e))))
+                .on('heartbeat', (_lsn, _time, respond) => {
+                respond && ack();
+            })
+                .on('data', (lsn, msg) => {
+                const change = messageToDownstream(lsn, msg);
+                if (change) {
+                    changes.push(change);
+                }
+            });
+            await this.stopExistingReplicationSlotSubscriber(db, slot);
+            this.#lc.info?.(`starting replication stream @${slot}`);
+            service
+                .subscribe(new PgoutputPlugin({
+                protoVersion: 1,
+                publicationNames: this.#replicationConfig.publications,
+            }), slot, fromLexiVersion(clientStart))
+                .catch(e => {
+                const err = translateError(e);
+                reject(err);
+                changes.fail(err);
+            })
+                .finally(() => changes.cancel());
+            const initialWatermark = await nextWatermark;
+            this.#lc.info?.(`replication stream@${slot} started at ${initialWatermark}`);
+            return { initialWatermark, changes, acks: { push: ack } };
+        }
+        finally {
+            await db.end();
+        }
+    }
+    async stopExistingReplicationSlotSubscriber(db, slot) {
+        const result = await db `
+    SELECT pg_terminate_backend(active_pid), active_pid as pid
+      FROM pg_replication_slots WHERE slot_name = ${slot} and active = true`;
+        if (result.length === 0) {
+            this.#lc.debug?.(`no existing subscriber to replication slot`);
+        }
+        else {
+            const { pid } = result[0];
+            this.#lc.info?.(`signaled subscriber ${pid} to shut down`);
+            // This reduces flakiness in which unit tests often fail with
+            // an error when starting the replication stream:
+            //
+            // error: replication slot "zero_slot_change_source_test_id" is active for PID 268
+            //
+            // Presumably, waiting for small interval before connecting to Postgres
+            // would also reduce this occurrence in production.
+            await sleep(5);
+        }
+    }
+    // Sometimes the `confirmed_flush_lsn` gets wiped, e.g.
+    //
+    // ```
+    //  slot_name | restart_lsn | confirmed_flush_lsn
+    //  -----------+-------------+---------------------
+    //  zero_slot | 8F/38ACB2F8 | 0/1
+    // ```
+    //
+    // Using the greatest of three values should always yield a correct result:
+    // * `clientWatermark`    : ahead of `confirmed_flush_lsn` if an ACK was lost,
+    //                          or if the `confirmed_flush_lsn` was wiped.
+    // * `confirmed_flush_lsn`: ahead of the `clientWatermark` if the ChangeDB was wiped.
+    // * `restart_lsn`        : if both the `confirmed_flush_lsn` and ChangeDB were wiped.
+    async getNextWatermark(db, slot, clientStart) {
+        const result = await db `
+      SELECT restart_lsn as restart, confirmed_flush_lsn as confirmed FROM pg_replication_slots 
+        WHERE slot_name = ${slot}`;
+        if (result.length === 0) {
+            throw new Error(`Upstream is missing replication slot ${slot}`);
+        }
+        const { restart, confirmed } = result[0];
+        const confirmedWatermark = toLexiVersion(confirmed);
+        const restartWatermark = toLexiVersion(restart);
+        this.#lc.info?.(`confirmed_flush_lsn:${confirmed}, restart_lsn:${restart}, clientWatermark:${fromLexiVersion(clientStart)}`);
+        return max(oneAfter(confirmedWatermark), oneAfter(restartWatermark), clientStart);
+    }
+}
+function messageToDownstream(lsn, msg) {
+    const change = msg;
+    const { tag } = change;
+    switch (tag) {
+        case 'begin':
+            return ['begin', change];
+        case 'insert':
+        case 'update':
+        case 'delete':
+        case 'truncate':
+            return ['data', change];
+        case 'commit': {
+            const watermark = toLexiVersion(lsn);
+            return ['commit', change, { watermark }];
+        }
+        default:
+            change; // All Change types are covered.
+            // But we can technically receive other Message types.
+            switch (msg.tag) {
+                case 'relation':
+                    return undefined; // Explicitly ignored. Schema handling is TODO.
+                case 'type':
+                    throw new Error(`Custom types are not supported (received "${msg.typeName}")`);
+                case 'origin':
+                    // We do not set the `origin` option in the pgoutput parameters:
+                    // https://www.postgresql.org/docs/current/protocol-logical-replication.html#PROTOCOL-LOGICAL-REPLICATION-PARAMS
+                    throw new Error(`Unexpected ORIGIN message ${stringify(msg)}`);
+                case 'message':
+                    // We do not set the `messages` option in the pgoutput parameters:
+                    // https://www.postgresql.org/docs/current/protocol-logical-replication.html#PROTOCOL-LOGICAL-REPLICATION-PARAMS
+                    throw new Error(`Unexpected MESSAGE message ${stringify(msg)}`);
+                default:
+                    throw new Error(`Unexpected message type ${stringify(msg)}`);
+            }
+    }
+}
+function translateError(e) {
+    if (!(e instanceof Error)) {
+        return new Error(String(e));
+    }
+    if (e instanceof DatabaseError && e.code === PG_ADMIN_SHUTDOWN) {
+        return new AbortError(e.message, { cause: e });
+    }
+    return e;
+}
+//# sourceMappingURL=change-source.js.map

package/out/zero-cache/src/services/change-streamer/pg/change-source.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-source.js","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/pg/change-source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAC,MAAM,gCAAgC,CAAC;AACjE,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,QAAQ,EAAC,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EACL,yBAAyB,EACzB,QAAQ,EACR,cAAc,GACf,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAC,aAAa,EAAC,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAC,UAAU,EAAC,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAC,SAAS,EAAC,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAC,KAAK,EAAC,MAAM,qBAAqB,CAAC;AAC1C,OAAO,EAAC,eAAe,EAAC,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAC,SAAS,EAAC,MAAM,qCAAqC,CAAC;AAC9D,OAAO,EAAC,GAAG,EAAE,QAAQ,EAAC,MAAM,sCAAsC,CAAC;AACnE,OAAO,EACL,QAAQ,EACR,2BAA2B,GAE5B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAC,YAAY,EAAC,MAAM,sCAAsC,CAAC;AAClE,OAAO,EAAC,QAAQ,EAAC,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAC,oBAAoB,EAAC,MAAM,8CAA8C,CAAC;AAKlF,OAAO,EAAC,eAAe,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAC,eAAe,EAAE,aAAa,EAAC,MAAM,UAAU,CAAC;AACxD,OAAO,EAAC,2BAA2B,EAAC,MAAM,kBAAkB,CAAC;AAE7D,OAAO,EAAC,cAAc,EAAC,MAAM,kBAAkB,CAAC;AAEhD,iDAAiD;AACjD,2BAA2B,EAAE,CAAC;AAE9B;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,EAAc,EACd,WAAmB,EACnB,KAAkB,EAClB,aAAqB;IAErB,MAAM,cAAc,CAClB,EAAE,EACF,iBAAiB,EACjB,KAAK,EACL,aAAa,EACb,WAAW,CACZ,CAAC;IAEF,MAAM,OAAO,GAAG,IAAI,QAAQ,CAAC,EAAE,EAAE,aAAa,CAAC,CAAC;IAChD,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC;IAE7E,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;QAC9B,2DAA2D;QAC3D,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC;QACjD,MAAM,UAAU,GAAG,iBAAiB,CAAC,YAAY;aAC9C,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;aACvD,IAAI,EAAE,CAAC;QACV,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,UAAU,CAAC,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CACb,gDAAgD,SAAS,wCAAwC,UAAU,GAAG,CAC/G,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,oBAAoB,CAC3C,EAAE,EACF,WAAW,EACX,KAAK,CAAC,EAAE,EACR,iBAAiB,CAClB,CAAC;IAEF,OAAO,EAAC,iBAAiB,EAAE,YAAY,EAAC,CAAC;AAC3C,CAAC;AAED;;;GAGG;AACH,MAAM,oBAAoB;IACf,GAAG,CAAa;IAChB,YAAY,CAAS;IACrB,QAAQ,CAAS;IACjB,kBAAkB,CAAoB;IAE/C,YACE,EAAc,EACd,WAAmB,EACnB,OAAe,EACf,iBAAoC;QAEpC,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC,WAAW,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QAChC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,IAAI,CAAC,kBAAkB,GAAG,iBAAiB,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,eAAuB;QACvC,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QACjD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,WAAW,GAAG,QAAQ,CAAC,eAAe,CAAC,CAAC;QAE9C,IAAI,CAAC;YACH,IAAI,OAAO,GAAG,KAAK,CAAC;YAEpB,MAAM,GAAG,GAAG,CAAC,MAAe,EAAE,EAAE;gBAC9B,IAAI,MAAM,EAAE,CAAC;oBACX,MAAM,EAAC,SAAS,EAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;oBAC9B,OAAO,GAAG,eAAe,CAAC,SAAS,CAAC,CAAC;gBACvC,CAAC;gBACD,KAAK,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC,CAAC;YAEF,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAmB;gBACpD,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE;aAC9B,CAAC,CAAC;YAEH,oEAAoE;YACpE,qEAAqE;YACrE,oEAAoE;YACpE,4DAA4D;YAC5D,MAAM,EAAC,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,EAAC,GAAG,QAAQ,EAAU,CAAC;YAErE,MAAM,OAAO,GAAG,IAAI,yBAAyB,CAC3C,EAAC,gBAAgB,EAAE,IAAI,CAAC,YAAY,EAAC,EACrC,EAAC,WAAW,EAAE,EAAC,IAAI,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,EAAC,EAAC,CAChD;iBACE,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAChB,IAAI,CAAC,gBAAgB,CAAC,EAAE,EAAE,IAAI,EAAE,WAAW,CAAC;iBACzC,IAAI,CAAC,OAAO,CAAC;iBACb,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CACzC;iBACA,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;gBACxC,OAAO,IAAI,GAAG,EAAE,CAAC;YACnB,CAAC,CAAC;iBACD,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;gBACvB,MAAM,MAAM,GAAG,mBAAmB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBAC7C,IAAI,MAAM,EAAE,CAAC;oBACX,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC,CAAC,CAAC;YAEL,MAAM,IAAI,CAAC,qCAAqC,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;YAE3D,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,gCAAgC,IAAI,EAAE,CAAC,CAAC;YACxD,OAAO;iBACJ,SAAS,CACR,IAAI,cAAc,CAAC;gBACjB,YAAY,EAAE,CAAC;gBACf,gBAAgB,EAAE,IAAI,CAAC,kBAAkB,CAAC,YAAY;aACvD,CAAC,EACF,IAAI,EACJ,eAAe,CAAC,WAAW,CAAC,CAC7B;iBACA,KAAK,CAAC,CAAC,CAAC,EAAE;gBACT,MAAM,GAAG,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;gBAC9B,MAAM,CAAC,GAAG,CAAC,CAAC;gBACZ,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAEnC,MAAM,gBAAgB,GAAG,MAAM,aAAa,CAAC;YAC7C,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,sBAAsB,IAAI,eAAe,gBAAgB,EAAE,CAC5D,CAAC;YAEF,OAAO,EAAC,gBAAgB,EAAE,OAAO,EAAE,IAAI,EAAE,EAAC,IAAI,EAAE,GAAG,EAAC,EAAC,CAAC;QACxD,CAAC;gBAAS,CAAC;YACT,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,qCAAqC,CACzC,EAAc,EACd,IAAY;QAEZ,MAAM,MAAM,GAAG,MAAM,EAAE,CAAiB;;oDAEQ,IAAI,oBAAoB,CAAC;QACzE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,4CAA4C,CAAC,CAAC;QACjE,CAAC;aAAM,CAAC;YACN,MAAM,EAAC,GAAG,EAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACxB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,uBAAuB,GAAG,eAAe,CAAC,CAAC;YAE3D,6DAA6D;YAC7D,iDAAiD;YACjD,EAAE;YACF,kFAAkF;YAClF,EAAE;YACF,uEAAuE;YACvE,mDAAmD;YACnD,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,EAAE;IACF,MAAM;IACN,iDAAiD;IACjD,mDAAmD;IACnD,iCAAiC;IACjC,MAAM;IACN,EAAE;IACF,2EAA2E;IAC3E,8EAA8E;IAC9E,sEAAsE;IACtE,qFAAqF;IACrF,sFAAsF;IACtF,KAAK,CAAC,gBAAgB,CACpB,EAAc,EACd,IAAY,EACZ,WAAmB;QAEnB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAwC;;4BAEvC,IAAI,EAAE,CAAC;QAC/B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,EAAE,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,EAAC,OAAO,EAAE,SAAS,EAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,kBAAkB,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;QACpD,MAAM,gBAAgB,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;QAEhD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,uBAAuB,SAAS,iBAAiB,OAAO,qBAAqB,eAAe,CAC1F,WAAW,CACZ,EAAE,CACJ,CAAC;QACF,OAAO,GAAG,CACR,QAAQ,CAAC,kBAAkB,CAAC,EAC5B,QAAQ,CAAC,gBAAgB,CAAC,EAC1B,WAAW,CACZ,CAAC;IACJ,CAAC;CACF;AAED,SAAS,mBAAmB,CAC1B,GAAW,EACX,GAAqB;IAErB,MAAM,MAAM,GAAG,GAAa,CAAC;IAC7B,MAAM,EAAC,GAAG,EAAC,GAAG,MAAM,CAAC;IACrB,QAAQ,GAAG,EAAE,CAAC;QACZ,KAAK,OAAO;YACV,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC3B,KAAK,QAAQ,CAAC;QACd,KAAK,QAAQ,CAAC;QACd,KAAK,QAAQ,CAAC;QACd,KAAK,UAAU;YACb,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC1B,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,SAAS,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;YACrC,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAC,SAAS,EAAC,CAAC,CAAC;QACzC,CAAC;QAED;YACE,MAAsB,CAAC,CAAC,gCAAgC;YAExD,sDAAsD;YACtD,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC;gBAChB,KAAK,UAAU;oBACb,OAAO,SAAS,CAAC,CAAC,+CAA+C;gBACnE,KAAK,MAAM;oBACT,MAAM,IAAI,KAAK,CACb,6CAA6C,GAAG,CAAC,QAAQ,IAAI,CAC9D,CAAC;gBACJ,KAAK,QAAQ;oBACX,gEAAgE;oBAChE,gHAAgH;oBAChH,MAAM,IAAI,KAAK,CAAC,6BAA6B,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACjE,KAAK,SAAS;oBACZ,kEAAkE;oBAClE,gHAAgH;oBAChH,MAAM,IAAI,KAAK,CAAC,8BAA8B,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAClE;oBACE,MAAM,IAAI,KAAK,CAAC,2BAA2B,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjE,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,CAAU;IAChC,IAAI,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,CAAC;IACD,IAAI,CAAC,YAAY,aAAa,IAAI,CAAC,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;QAC/D,OAAO,IAAI,UAAU,CAAC,CAAC,CAAC,OAAO,EAAE,EAAC,KAAK,EAAE,CAAC,EAAC,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC"}

package/out/zero-cache/src/services/change-streamer/pg/initial-sync.d.ts

@@ -0,0 +1,6 @@
+import type { LogContext } from '@rocicorp/logger';
+import { Database } from '../../../../../zqlite/src/db.js';
+import type { ShardConfig } from './shard-config.js';
+export declare function replicationSlot(shardID: string): string;
+export declare function initialSync(lc: LogContext, shard: ShardConfig, tx: Database, upstreamURI: string): Promise<void>;
+//# sourceMappingURL=initial-sync.d.ts.map

package/out/zero-cache/src/services/change-streamer/pg/initial-sync.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"initial-sync.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/pg/initial-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAYjD,OAAO,EAAC,QAAQ,EAAC,MAAM,kBAAkB,CAAC;AAW1C,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,mBAAmB,CAAC;AAEnD,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAEvD;AAID,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,iBAoDpB"}

package/out/zero-cache/src/services/change-streamer/pg/initial-sync.js

@@ -0,0 +1,192 @@
+import { ident } from 'pg-format';
+import postgres from 'postgres';
+import { importSnapshot, Mode, TransactionPool, } from '../../../../../zero-cache/src/db/transaction-pool.js';
+import { liteValues } from '../../../../../zero-cache/src/types/lite.js';
+import { liteTableName } from '../../../../../zero-cache/src/types/names.js';
+import { pgClient } from '../../../../../zero-cache/src/types/pg.js';
+import { Database } from '../../../../../zqlite/src/db.js';
+import { initChangeLog } from '../../replicator/schema/change-log.js';
+import { initReplicationState, ZERO_VERSION_COLUMN_NAME, } from '../../replicator/schema/replication-state.js';
+import { toLexiVersion } from './lsn.js';
+import { createTableStatement } from './schema/create.js';
+import { checkDataTypeSupported, mapPostgresToLite } from './schema/lite.js';
+import {} from './schema/published.js';
+import { setupTablesAndReplication } from './schema/zero.js';
+export function replicationSlot(shardID) {
+    return `zero_${shardID}`;
+}
+const ALLOWED_IDENTIFIER_CHARS = /^[A-Za-z_-]+$/;
+export async function initialSync(lc, shard, tx, upstreamURI) {
+    const upstreamDB = pgClient(lc, upstreamURI, {
+        max: MAX_WORKERS,
+    });
+    const replicationSession = pgClient(lc, upstreamURI, {
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        fetch_types: false, // Necessary for the streaming protocol
+        connection: { replication: 'database' }, // https://www.postgresql.org/docs/current/protocol-replication.html
+    });
+    try {
+        await checkUpstreamConfig(upstreamDB);
+        const { publications, tables, indices } = await ensurePublishedTables(lc, upstreamDB, shard);
+        const pubNames = publications.map(p => p.pubname);
+        lc.info?.(`Upstream is setup with publications [${pubNames}]`);
+        createLiteTables(tx, tables);
+        createLiteIndices(tx, indices);
+        const { database, host } = upstreamDB.options;
+        lc.info?.(`opening replication session to ${database}@${host}`);
+        const { snapshot_name: snapshot, consistent_point: lsn } = await createReplicationSlot(lc, shard.id, replicationSession);
+        // Run up to MAX_WORKERS to copy of tables at the replication slot's snapshot.
+        const copiers = startTableCopyWorkers(lc, upstreamDB, tables.length, snapshot);
+        await Promise.all(tables.map(table => copiers.processReadTask(db => copy(lc, table, db, tx))));
+        copiers.setDone();
+        initReplicationState(tx, pubNames, toLexiVersion(lsn));
+        initChangeLog(tx);
+        lc.info?.(`Synced initial data from ${pubNames} up to ${lsn}`);
+        await copiers.done();
+    }
+    finally {
+        // Close the upstream connections.
+        await replicationSession.end();
+        await upstreamDB.end();
+    }
+}
+async function checkUpstreamConfig(upstreamDB) {
+    const { walLevel, version } = (await upstreamDB `
+      SELECT current_setting('wal_level') as "walLevel", 
+             current_setting('server_version_num') as "version";
+  `)[0];
+    if (walLevel !== 'logical') {
+        throw new Error(`Postgres must be configured with "wal_level = logical" (currently: "${walLevel})`);
+    }
+    if (version < 150000) {
+        throw new Error(`Must be running Postgres 15 or higher (currently: "${version}")`);
+    }
+}
+function ensurePublishedTables(lc, upstreamDB, shard) {
+    const { database, host } = upstreamDB.options;
+    lc.info?.(`Ensuring upstream PUBLICATION on ${database}@${host}`);
+    return upstreamDB.begin(async (tx) => {
+        const published = await setupTablesAndReplication(tx, shard);
+        // Verify that all publications export the proper events.
+        published.publications.forEach(pub => {
+            if (!pub.pubinsert ||
+                !pub.pubtruncate ||
+                !pub.pubdelete ||
+                !pub.pubtruncate) {
+                // TODO: Make APIError?
+                throw new Error(`PUBLICATION ${pub.pubname} must publish insert, update, delete, and truncate`);
+            }
+        });
+        published.tables.forEach(table => {
+            if (!['public', 'zero'].includes(table.schema)) {
+                // This may be relaxed in the future. We would need a plan for support in the AST first.
+                throw new Error('Only the default "public" schema is supported.');
+            }
+            if (ZERO_VERSION_COLUMN_NAME in table.columns) {
+                throw new Error(`Table "${table.name}" uses reserved column name "${ZERO_VERSION_COLUMN_NAME}"`);
+            }
+            if (table.primaryKey.length === 0) {
+                throw new Error(`Table "${table.name}" does not have a PRIMARY KEY`);
+            }
+            if (!ALLOWED_IDENTIFIER_CHARS.test(table.schema)) {
+                throw new Error(`Schema "${table.schema}" has invalid characters.`);
+            }
+            if (!ALLOWED_IDENTIFIER_CHARS.test(table.name)) {
+                throw new Error(`Table "${table.name}" has invalid characters.`);
+            }
+            for (const [col, spec] of Object.entries(table.columns)) {
+                if (!ALLOWED_IDENTIFIER_CHARS.test(col)) {
+                    throw new Error(`Column "${col}" in table "${table.name}" has invalid characters.`);
+                }
+                checkDataTypeSupported(spec.dataType);
+            }
+        });
+        return published;
+    });
+}
+/* eslint-enable @typescript-eslint/naming-convention */
+async function createReplicationSlot(lc, shardID, session) {
+    // Note: The replication connection does not support the extended query protocol,
+    //       so all commands must be sent using sql.unsafe(). This is technically safe
+    //       because all placeholder values are under our control (i.e. "slotName").
+    const slotName = replicationSlot(shardID);
+    const slots = await session.unsafe(`SELECT * FROM pg_replication_slots WHERE slot_name = '${slotName}'`);
+    // Because a snapshot created by CREATE_REPLICATION_SLOT only lasts for the lifetime
+    // of the replication session, if there is an existing slot, it must be deleted so that
+    // the slot (and corresponding snapshot) can be created anew.
+    //
+    // This means that in order for initial data sync to succeed, it must fully complete
+    // within the lifetime of a replication session. Note that this is same requirement
+    // (and behavior) for Postgres-to-Postgres initial sync:
+    // https://github.com/postgres/postgres/blob/5304fec4d8a141abe6f8f6f2a6862822ec1f3598/src/backend/replication/logical/tablesync.c#L1358
+    if (slots.length > 0) {
+        lc.info?.(`Dropping existing replication slot ${slotName}`);
+        await session.unsafe(`DROP_REPLICATION_SLOT ${slotName} WAIT`);
+    }
+    const slot = (await session.unsafe(`CREATE_REPLICATION_SLOT ${slotName} LOGICAL pgoutput`))[0];
+    lc.info?.(`Created replication slot ${slotName}`, slot);
+    return slot;
+}
+// TODO: Consider parameterizing these.
+const MAX_WORKERS = 5;
+const BATCH_SIZE = 100_000;
+function startTableCopyWorkers(lc, db, numTables, snapshot) {
+    const { init } = importSnapshot(snapshot);
+    const numWorkers = Math.min(numTables, MAX_WORKERS);
+    const tableCopiers = new TransactionPool(lc, Mode.READONLY, init, undefined, numWorkers);
+    tableCopiers.run(db);
+    lc.info?.(`Started ${numWorkers} workers to copy ${numTables} tables`);
+    return tableCopiers;
+}
+function createLiteTables(tx, tables) {
+    for (const t of tables) {
+        const liteTable = mapPostgresToLite(t);
+        tx.exec(createTableStatement(liteTable));
+    }
+}
+function createLiteIndices(tx, indices) {
+    for (const index of indices) {
+        const tableName = liteTableName({
+            schema: index.schemaName,
+            name: index.tableName,
+        });
+        const columns = index.columns.map(c => ident(c)).join(',');
+        const unique = index.unique ? 'UNIQUE ' : '';
+        tx.exec(`CREATE ${unique} INDEX ${ident(index.name)} ON ${ident(tableName)} (${columns})`);
+    }
+}
+async function copy(lc, table, from, to) {
+    let totalRows = 0;
+    const tableName = liteTableName(table);
+    const selectColumns = Object.keys(table.columns)
+        .map(c => ident(c))
+        .join(',');
+    const insertColumns = [
+        ...Object.keys(table.columns),
+        ZERO_VERSION_COLUMN_NAME,
+    ];
+    const insertColumnList = insertColumns.map(c => ident(c)).join(',');
+    const insertStmt = to.prepare(`INSERT INTO "${tableName}" (${insertColumnList}) VALUES (${new Array(insertColumns.length)
+        .fill('?')
+        .join(',')})`);
+    const filterConditions = Object.values(table.publications)
+        .map(({ rowFilter }) => rowFilter)
+        .filter(f => !!f); // remove nulls
+    const selectStmt = `SELECT ${selectColumns} FROM ${ident(table.schema)}.${ident(table.name)}` +
+        (filterConditions.length === 0
+            ? ''
+            : ` WHERE ${filterConditions.join(' OR ')}`);
+    const cursor = from.unsafe(selectStmt).cursor(BATCH_SIZE);
+    for await (const rows of cursor) {
+        for (const row of rows) {
+            insertStmt.run([
+                ...liteValues(row),
+                '00', // initial _0_version
+            ]);
+        }
+        totalRows += rows.length;
+        lc.debug?.(`Copied ${totalRows} rows from ${table.schema}.${table.name}`);
+    }
+    lc.info?.(`Finished copying ${totalRows} rows into ${tableName}`);
+}
+//# sourceMappingURL=initial-sync.js.map

package/out/zero-cache/src/services/change-streamer/pg/initial-sync.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"initial-sync.js","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/pg/initial-sync.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,KAAK,EAAC,MAAM,WAAW,CAAC;AAChC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EACL,cAAc,EACd,IAAI,EACJ,eAAe,GAChB,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EAAC,UAAU,EAAC,MAAM,8BAA8B,CAAC;AACxD,OAAO,EAAC,aAAa,EAAC,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAC,QAAQ,EAAkB,MAAM,4BAA4B,CAAC;AAErE,OAAO,EAAC,QAAQ,EAAC,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAC,aAAa,EAAC,MAAM,uCAAuC,CAAC;AACpE,OAAO,EACL,oBAAoB,EACpB,wBAAwB,GACzB,MAAM,8CAA8C,CAAC;AACtD,OAAO,EAAC,aAAa,EAAC,MAAM,UAAU,CAAC;AACvC,OAAO,EAAC,oBAAoB,EAAC,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAC,sBAAsB,EAAE,iBAAiB,EAAC,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAsB,MAAM,uBAAuB,CAAC;AAC3D,OAAO,EAAC,yBAAyB,EAAC,MAAM,kBAAkB,CAAC;AAG3D,MAAM,UAAU,eAAe,CAAC,OAAe;IAC7C,OAAO,QAAQ,OAAO,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,wBAAwB,GAAG,eAAe,CAAC;AAEjD,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,EAAc,EACd,KAAkB,EAClB,EAAY,EACZ,WAAmB;IAEnB,MAAM,UAAU,GAAG,QAAQ,CAAC,EAAE,EAAE,WAAW,EAAE;QAC3C,GAAG,EAAE,WAAW;KACjB,CAAC,CAAC;IACH,MAAM,kBAAkB,GAAG,QAAQ,CAAC,EAAE,EAAE,WAAW,EAAE;QACnD,gEAAgE;QAChE,WAAW,EAAE,KAAK,EAAE,uCAAuC;QAC3D,UAAU,EAAE,EAAC,WAAW,EAAE,UAAU,EAAC,EAAE,oEAAoE;KAC5G,CAAC,CAAC;IACH,IAAI,CAAC;QACH,MAAM,mBAAmB,CAAC,UAAU,CAAC,CAAC;QACtC,MAAM,EAAC,YAAY,EAAE,MAAM,EAAE,OAAO,EAAC,GAAG,MAAM,qBAAqB,CACjE,EAAE,EACF,UAAU,EACV,KAAK,CACN,CAAC;QACF,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAClD,EAAE,CAAC,IAAI,EAAE,CAAC,wCAAwC,QAAQ,GAAG,CAAC,CAAC;QAE/D,gBAAgB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAC7B,iBAAiB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAE/B,MAAM,EAAC,QAAQ,EAAE,IAAI,EAAC,GAAG,UAAU,CAAC,OAAO,CAAC;QAC5C,EAAE,CAAC,IAAI,EAAE,CAAC,kCAAkC,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAC;QAChE,MAAM,EAAC,aAAa,EAAE,QAAQ,EAAE,gBAAgB,EAAE,GAAG,EAAC,GACpD,MAAM,qBAAqB,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,kBAAkB,CAAC,CAAC;QAEhE,8EAA8E;QAC9E,MAAM,OAAO,GAAG,qBAAqB,CACnC,EAAE,EACF,UAAU,EACV,MAAM,CAAC,MAAM,EACb,QAAQ,CACT,CAAC;QACF,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CACjB,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CACvD,CACF,CAAC;QACF,OAAO,CAAC,OAAO,EAAE,CAAC;QAElB,oBAAoB,CAAC,EAAE,EAAE,QAAQ,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,aAAa,CAAC,EAAE,CAAC,CAAC;QAClB,EAAE,CAAC,IAAI,EAAE,CAAC,4BAA4B,QAAQ,UAAU,GAAG,EAAE,CAAC,CAAC;QAE/D,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;YAAS,CAAC;QACT,kCAAkC;QAClC,MAAM,kBAAkB,CAAC,GAAG,EAAE,CAAC;QAC/B,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC;IACzB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,UAAsB;IACvD,MAAM,EAAC,QAAQ,EAAE,OAAO,EAAC,GAAG,CAC1B,MAAM,UAAU,CAAuC;;;GAGxD,CACA,CAAC,CAAC,CAAC,CAAC;IAEL,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,uEAAuE,QAAQ,GAAG,CACnF,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,GAAG,MAAM,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CACb,sDAAsD,OAAO,IAAI,CAClE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,EAAc,EACd,UAAsB,EACtB,KAAkB;IAElB,MAAM,EAAC,QAAQ,EAAE,IAAI,EAAC,GAAG,UAAU,CAAC,OAAO,CAAC;IAC5C,EAAE,CAAC,IAAI,EAAE,CAAC,oCAAoC,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAC;IAElE,OAAO,UAAU,CAAC,KAAK,CAAC,KAAK,EAAC,EAAE,EAAC,EAAE;QACjC,MAAM,SAAS,GAAG,MAAM,yBAAyB,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAC7D,yDAAyD;QACzD,SAAS,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YACnC,IACE,CAAC,GAAG,CAAC,SAAS;gBACd,CAAC,GAAG,CAAC,WAAW;gBAChB,CAAC,GAAG,CAAC,SAAS;gBACd,CAAC,GAAG,CAAC,WAAW,EAChB,CAAC;gBACD,uBAAuB;gBACvB,MAAM,IAAI,KAAK,CACb,eAAe,GAAG,CAAC,OAAO,oDAAoD,CAC/E,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;YAC/B,IAAI,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/C,wFAAwF;gBACxF,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;YACpE,CAAC;YACD,IAAI,wBAAwB,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC9C,MAAM,IAAI,KAAK,CACb,UAAU,KAAK,CAAC,IAAI,gCAAgC,wBAAwB,GAAG,CAChF,CAAC;YACJ,CAAC;YACD,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC,IAAI,+BAA+B,CAAC,CAAC;YACvE,CAAC;YACD,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;gBACjD,MAAM,IAAI,KAAK,CAAC,WAAW,KAAK,CAAC,MAAM,2BAA2B,CAAC,CAAC;YACtE,CAAC;YACD,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/C,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC,IAAI,2BAA2B,CAAC,CAAC;YACnE,CAAC;YACD,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACxD,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;oBACxC,MAAM,IAAI,KAAK,CACb,WAAW,GAAG,eAAe,KAAK,CAAC,IAAI,2BAA2B,CACnE,CAAC;gBACJ,CAAC;gBACD,sBAAsB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,SAAS,CAAC;IACnB,CAAC,CAAC,CAAC;AACL,CAAC;AAUD,wDAAwD;AAExD,KAAK,UAAU,qBAAqB,CAClC,EAAc,EACd,OAAe,EACf,OAAqB;IAErB,iFAAiF;IACjF,kFAAkF;IAClF,gFAAgF;IAChF,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,MAAM,CAChC,yDAAyD,QAAQ,GAAG,CACrE,CAAC;IAEF,oFAAoF;IACpF,uFAAuF;IACvF,6DAA6D;IAC7D,EAAE;IACF,oFAAoF;IACpF,mFAAmF;IACnF,wDAAwD;IACxD,uIAAuI;IACvI,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,EAAE,CAAC,IAAI,EAAE,CAAC,sCAAsC,QAAQ,EAAE,CAAC,CAAC;QAC5D,MAAM,OAAO,CAAC,MAAM,CAAC,yBAAyB,QAAQ,OAAO,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,IAAI,GAAG,CACX,MAAM,OAAO,CAAC,MAAM,CAClB,2BAA2B,QAAQ,mBAAmB,CACvD,CACF,CAAC,CAAC,CAAC,CAAC;IACL,EAAE,CAAC,IAAI,EAAE,CAAC,4BAA4B,QAAQ,EAAE,EAAE,IAAI,CAAC,CAAC;IACxD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,uCAAuC;AACvC,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,UAAU,GAAG,OAAO,CAAC;AAE3B,SAAS,qBAAqB,CAC5B,EAAc,EACd,EAAc,EACd,SAAiB,EACjB,QAAgB;IAEhB,MAAM,EAAC,IAAI,EAAC,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IACpD,MAAM,YAAY,GAAG,IAAI,eAAe,CACtC,EAAE,EACF,IAAI,CAAC,QAAQ,EACb,IAAI,EACJ,SAAS,EACT,UAAU,CACX,CAAC;IACF,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAErB,EAAE,CAAC,IAAI,EAAE,CAAC,WAAW,UAAU,oBAAoB,SAAS,SAAS,CAAC,CAAC;IACvE,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,SAAS,gBAAgB,CAAC,EAAY,EAAE,MAA2B;IACjE,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,SAAS,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;QACvC,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,EAAY,EAAE,OAAoB;IAC3D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAG,aAAa,CAAC;YAC9B,MAAM,EAAE,KAAK,CAAC,UAAU;YACxB,IAAI,EAAE,KAAK,CAAC,SAAS;SACtB,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3D,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,EAAE,CAAC,IAAI,CACL,UAAU,MAAM,UAAU,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CACrD,SAAS,CACV,KAAK,OAAO,GAAG,CACjB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI,CACjB,EAAc,EACd,KAAwB,EACxB,IAAgB,EAChB,EAAY;IAEZ,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACvC,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;SAC7C,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SAClB,IAAI,CAAC,GAAG,CAAC,CAAC;IACb,MAAM,aAAa,GAAG;QACpB,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;QAC7B,wBAAwB;KACzB,CAAC;IACF,MAAM,gBAAgB,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpE,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAC3B,gBAAgB,SAAS,MAAM,gBAAgB,aAAa,IAAI,KAAK,CACnE,aAAa,CAAC,MAAM,CACrB;SACE,IAAI,CAAC,GAAG,CAAC;SACT,IAAI,CAAC,GAAG,CAAC,GAAG,CAChB,CAAC;IACF,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC;SACvD,GAAG,CAAC,CAAC,EAAC,SAAS,EAAC,EAAE,EAAE,CAAC,SAAS,CAAC;SAC/B,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe;IACpC,MAAM,UAAU,GACd,UAAU,aAAa,SAAS,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;QAC1E,CAAC,gBAAgB,CAAC,MAAM,KAAK,CAAC;YAC5B,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,UAAU,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEjD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC1D,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAChC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,UAAU,CAAC,GAAG,CAAC;gBACb,GAAG,UAAU,CAAC,GAAG,CAAC;gBAClB,IAAI,EAAE,qBAAqB;aAC5B,CAAC,CAAC;QACL,CAAC;QACD,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC;QACzB,EAAE,CAAC,KAAK,EAAE,CAAC,UAAU,SAAS,cAAc,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5E,CAAC;IACD,EAAE,CAAC,IAAI,EAAE,CAAC,oBAAoB,SAAS,cAAc,SAAS,EAAE,CAAC,CAAC;AACpE,CAAC"}

package/out/zero-cache/src/services/change-streamer/pg/lsn.d.ts

@@ -0,0 +1,17 @@
+import { type LexiVersion } from '../../../../../zero-cache/src/types/lexi-version.js';
+import type { Change } from '../schema/change.js';
+/**
+ * Parsing and conversion utilities for the pg_lsn Type, which represents
+ * the "Log Sequence Number" used as a monotonic progress marker for logical
+ * replication from a PostgresSQL database.
+ *
+ * The LSN is a 64-bit integer represented in logical replication as two
+ * hexadecimal numbers (up to 8 digits each) separated by a slash. This is
+ * converted to a LexiVersion and used as DB-agnostic version in change log,
+ * invalidation index, and row version in the tables of the sync replica.
+ */
+export type LSN = string;
+export type RecordType = Change['tag'];
+export declare function toLexiVersion(lsn: LSN): LexiVersion;
+export declare function fromLexiVersion(lexi: LexiVersion): LSN;
+//# sourceMappingURL=lsn.d.ts.map

package/out/zero-cache/src/services/change-streamer/pg/lsn.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lsn.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/pg/lsn.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,KAAK,WAAW,EACjB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,qBAAqB,CAAC;AAEhD;;;;;;;;;GASG;AACH,MAAM,MAAM,GAAG,GAAG,MAAM,CAAC;AAEzB,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;AAEvC,wBAAgB,aAAa,CAAC,GAAG,EAAE,GAAG,GAAG,WAAW,CAOnD;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,WAAW,GAAG,GAAG,CAKtD"}

package/out/zero-cache/src/services/change-streamer/pg/lsn.js

@@ -0,0 +1,17 @@
+import { assert } from '../../../../../shared/src/asserts.js';
+import { versionFromLexi, versionToLexi, } from '../../../../../zero-cache/src/types/lexi-version.js';
+export function toLexiVersion(lsn) {
+    const parts = lsn.split('/');
+    assert(parts.length === 2, `Malformed LSN: "${lsn}"`);
+    const high = BigInt(`0x${parts[0]}`);
+    const low = BigInt(`0x${parts[1]}`);
+    const val = (high << 32n) + low;
+    return versionToLexi(val);
+}
+export function fromLexiVersion(lexi) {
+    const val = versionFromLexi(lexi);
+    const high = val >> 32n;
+    const low = val & 0xffffffffn;
+    return `${high.toString(16).toUpperCase()}/${low.toString(16).toUpperCase()}`;
+}
+//# sourceMappingURL=lsn.js.map

package/out/zero-cache/src/services/change-streamer/pg/lsn.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lsn.js","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/pg/lsn.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,uBAAuB,CAAC;AAC7C,OAAO,EACL,eAAe,EACf,aAAa,GAEd,MAAM,sCAAsC,CAAC;AAiB9C,MAAM,UAAU,aAAa,CAAC,GAAQ;IACpC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC7B,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,mBAAmB,GAAG,GAAG,CAAC,CAAC;IACtD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACrC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC;IAChC,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAiB;IAC/C,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,GAAG,IAAI,GAAG,CAAC;IACxB,MAAM,GAAG,GAAG,GAAG,GAAG,WAAW,CAAC;IAC9B,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,IAAI,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;AAChF,CAAC"}

package/out/zero-cache/src/services/change-streamer/pg/schema/create.d.ts

@@ -0,0 +1,6 @@
+import type { TableSpec } from '../../../../../../zero-cache/src/types/specs.js';
+/**
+ * Constructs a `CREATE TABLE` statement for a {@link TableSpec}.
+ */
+export declare function createTableStatement(spec: TableSpec): string;
+//# sourceMappingURL=create.d.ts.map

package/out/zero-cache/src/services/change-streamer/pg/schema/create.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-streamer/pg/schema/create.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAa,SAAS,EAAC,MAAM,+BAA+B,CAAC;AAGzE;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,CAyB5D"}