@rocicorp/zero 0.0.2024100800 → 0.0.202410040736

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -1,212 +0,0 @@
-import { LogContext } from '@rocicorp/logger';
-import { oneAfter } from '../../types/lexi-version.js';
-import { Subscription } from '../../types/subscription.js';
-import { RunningState } from '../running-state.js';
-import { ErrorType, } from './change-streamer.js';
-import { Forwarder } from './forwarder.js';
-import { initChangeStreamerSchema } from './schema/init.js';
-import { ensureReplicationConfig, } from './schema/tables.js';
-import { Storer } from './storer.js';
-import { Subscriber } from './subscriber.js';
-/**
- * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.
- */
-export async function initializeStreamer(lc, changeDB, changeSource, replicationConfig) {
-    // Make sure the ChangeLog DB is set up.
-    await initChangeStreamerSchema(lc, changeDB);
-    await ensureReplicationConfig(lc, changeDB, replicationConfig);
-    const { replicaVersion } = replicationConfig;
-    return new ChangeStreamerImpl(lc, changeDB, replicaVersion, changeSource);
-}
-/**
- * Upstream-agnostic dispatch of messages in a {@link ChangeStream} to a
- * {@link Forwarder} and {@link Storer} to execute the forward-store-ack
- * procedure described in {@link ChangeStreamer}.
- *
- * ### Subscriber Catchup
- *
- * Connecting clients first need to be "caught up" to the current watermark
- * (from stored change log entries) before new entries are forwarded to
- * them. This is non-trivial because the replication stream may be in the
- * middle of a pending streamed Transaction for which some entries have
- * already been forwarded but are not yet committed to the store.
- *
- *
- * ```
- * ------------------------------- - - - - - - - - - - - - - - - - - - -
- * | Historic changes in storage |  Pending (streamed) tx  |   Next tx
- * ------------------------------- - - - - - - - - - - - - - - - - - - -
- *                                           Replication stream
- *                                           >  >  >  >  >  >  >  >  >
- *           ^  ---> required catchup --->   ^
- * Subscriber watermark               Subscription begins
- * ```
- *
- * Preemptively buffering the changes of every pending transaction
- * would be wasteful and consume too much memory for large transactions.
- *
- * Instead, the streamer synchronously dispatches changes and subscriptions
- * to the {@link Forwarder} and the {@link Storer} such that the two
- * components are aligned as to where in the stream the subscription started.
- * The two components then coordinate catchup and handoff via the
- * {@link Subscriber} object with the following algorithm:
- *
- * * If the streamer is in the middle of a pending Transaction, the
- *   Subscriber is "queued" on both the Forwarder and the Storer. In this
- *   state, new changes are *not* forwarded to the Subscriber, and catchup
- *   is not yet executed.
- * * Once the commit message for the pending Transaction is processed
- *   by the Storer, it begins catchup on the Subscriber (with a READONLY
- *   snapshot so that it does not block subsequent storage operations).
- *   This catchup is thus guaranteed to load the change log entries of
- *   that last Transaction.
- * * When the Forwarder processes that same commit message, it moves the
- *   Subscriber from the "queued" to the "active" set of clients such that
- *   the Subscriber begins receiving new changes, starting from the next
- *   Transaction.
- * * The Subscriber does not forward those changes, however, if its catchup
- *   is not complete. Until then, it buffers the changes in memory.
- * * Once catchup is complete, the buffered changes are immediately sent
- *   and the Subscriber henceforth forwards changes as they are received.
- *
- * In the (common) case where the streamer is not in the middle of a pending
- * transaction when a subscription begins, the Storer begins catchup
- * immediately and the Forwarder directly adds the Subscriber to its active
- * set. However, the Subscriber still buffers any forwarded messages until
- * its catchup is complete.
- *
- * ### Watermarks and ordering
- *
- * The ChangeStreamerService depends on its {@link ChangeSource} to send
- * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences
- * in commit order. This follows Postgres's Logical Replication Protocol
- * Message Flow:
- *
- * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW
- *
- * > The logical replication protocol sends individual transactions one by one.
- * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.
- *
- * In order to correctly replay (new) and filter (old) messages to subscribers
- * at different points in the replication stream, these changes must be assigned
- * watermarks such that they preserve the order in which they were received
- * from the ChangeSource.
- *
- * A previous implementation incorrectly derived these watermarks from the Postgres
- * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,
- * non-conflicting transactions can overlap, which can result in a `begin` message
- * with an earlier LSN arriving after a `commit` message. For example, the
- * changes for these transactions:
- *
- * ```
- * LSN:   1     2     3  4    5   6   7     8   9      10
- * tx1: begin  data     data     data     commit
- * tx2:             begin    data    data      data  commit
- * ```
- *
- * will arrive as:
- *
- * ```
- * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10
- * ```
- *
- * Thus, LSN of non-commit messages are not suitable for tracking the sorting
- * order of the replication stream.
- *
- * Instead, the ChangeStreamer uses the following algorithm for deterministic
- * catchup and filtering of changes:
- *
- * * A `commit` message is assigned to a watermark corresponding to its LSN.
- *   These are guaranteed to be in commit order by definition.
- *
- * * `begin` and `data` messages are assigned to the watermark of the
- *   preceding `commit` (the previous transaction, or the replication
- *   slot's starting LSN) plus 1. This guarantees that they will be sorted
- *   after the previously commit transaction even if their LSNs came before it.
- *   This is referred to as the `preCommitWatermark`.
- *
- * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is
- *   the position of the message within its transaction, with the `begin` message
- *   starting at `0`. This guarantees that `begin` and `data` messages will be
- *   fetched in the original ChangeSource order during catchup.
- *
- * `begin` and `data` messages share the same watermark, but this is sufficient for
- * Subscriber filtering because subscribers only know about the `commit` watermarks
- * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares
- * the internal watermarks of the incoming messages against the commit watermark of
- * the caller, updating the watermark at every `Commit` message that is forwarded.
- */
-class ChangeStreamerImpl {
-    id;
-    #lc;
-    #replicaVersion;
-    #source;
-    #storer;
-    #forwarder;
-    #state = new RunningState('ChangeStreamer');
-    #stream;
-    constructor(lc, changeDB, replicaVersion, source) {
-        this.id = `change-streamer`;
-        this.#lc = lc.withContext('component', 'change-streamer');
-        this.#replicaVersion = replicaVersion;
-        this.#source = source;
-        this.#storer = new Storer(lc, changeDB, commit => this.#stream?.acks.push(commit));
-        this.#forwarder = new Forwarder();
-    }
-    async run() {
-        this.#storer.run().catch(e => this.stop(e));
-        while (this.#state.shouldRun()) {
-            let err;
-            try {
-                const startAfter = await this.#storer.getLastStoredWatermark();
-                const stream = await this.#source.startStream(startAfter ?? this.#replicaVersion);
-                this.#stream = stream;
-                let preCommitWatermark = stream.initialWatermark;
-                for await (const change of stream.changes) {
-                    this.#state.resetBackoff();
-                    let watermark;
-                    if (change[0] !== 'commit') {
-                        watermark = preCommitWatermark;
-                    }
-                    else {
-                        watermark = change[2].watermark;
-                        preCommitWatermark = oneAfter(watermark); // For the next transaction.
-                    }
-                    this.#storer.store([watermark, change]);
-                    this.#forwarder.forward([watermark, change]);
-                }
-            }
-            catch (e) {
-                err = e;
-            }
-            finally {
-                this.#stream?.changes.cancel();
-                this.#stream = undefined;
-            }
-            await this.#state.backoff(this.#lc, err);
-        }
-        this.#lc.info?.('ChangeStreamer stopped');
-    }
-    subscribe(ctx) {
-        const { id, watermark } = ctx;
-        const downstream = Subscription.create({
-            cleanup: () => this.#forwarder.remove(subscriber),
-        });
-        const subscriber = new Subscriber(id, watermark, downstream);
-        if (ctx.replicaVersion !== this.#replicaVersion) {
-            subscriber.close(ErrorType.WrongReplicaVersion);
-        }
-        else {
-            this.#lc.debug?.(`adding subscriber ${subscriber.id}`);
-            this.#forwarder.add(subscriber);
-            this.#storer.catchup(subscriber);
-        }
-        return downstream;
-    }
-    async stop(err) {
-        this.#state.stop(this.#lc, err);
-        this.#stream?.changes.cancel();
-        await this.#storer.stop();
-    }
-}
-//# sourceMappingURL=change-streamer-service.js.map

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"change-streamer-service.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,QAAQ,EAAC,MAAM,6BAA6B,CAAC;AAGrD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAKL,SAAS,GAEV,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,SAAS,EAAC,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAC,wBAAwB,EAAC,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EACL,uBAAuB,GAExB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;AACnC,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAE3C;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,EAAc,EACd,QAAoB,EACpB,YAA0B,EAC1B,iBAAoC;IAEpC,wCAAwC;IACxC,MAAM,wBAAwB,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC7C,MAAM,uBAAuB,CAAC,EAAE,EAAE,QAAQ,EAAE,iBAAiB,CAAC,CAAC;IAE/D,MAAM,EAAC,cAAc,EAAC,GAAG,iBAAiB,CAAC;IAC3C,OAAO,IAAI,kBAAkB,CAAC,EAAE,EAAE,QAAQ,EAAE,cAAc,EAAE,YAAY,CAAC,CAAC;AAC5E,CAAC;AAoCD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqHG;AACH,MAAM,kBAAkB;IACb,EAAE,CAAS;IACX,GAAG,CAAa;IAChB,eAAe,CAAS;IACxB,OAAO,CAAe;IACtB,OAAO,CAAS;IAChB,UAAU,CAAY;IAEtB,MAAM,GAAG,IAAI,YAAY,CAAC,gBAAgB,CAAC,CAAC;IACrD,OAAO,CAA2B;IAElC,YACE,EAAc,EACd,QAAoB,EACpB,cAAsB,EACtB,MAAoB;QAEpB,IAAI,CAAC,EAAE,GAAG,iBAAiB,CAAC;QAC5B,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC,WAAW,CAAC,WAAW,EAAE,iBAAiB,CAAC,CAAC;QAC1D,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,OAAO,GAAG,IAAI,MAAM,CACvB,EAAE,EACF,QAAQ,EACR,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAC1C,CAAC;QACF,IAAI,CAAC,UAAU,GAAG,IAAI,SAAS,EAAE,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,GAAG;QACP,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5C,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC;YAC/B,IAAI,GAAY,CAAC;YACjB,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;gBAC/D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAC3C,UAAU,IAAI,IAAI,CAAC,eAAe,CACnC,CAAC;gBACF,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;gBACtB,IAAI,kBAAkB,GAAG,MAAM,CAAC,gBAAgB,CAAC;gBAEjD,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC1C,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;oBAE3B,IAAI,SAAiB,CAAC;oBACtB,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;wBAC3B,SAAS,GAAG,kBAAkB,CAAC;oBACjC,CAAC;yBAAM,CAAC;wBACN,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;wBAChC,kBAAkB,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B;oBACxE,CAAC;oBAED,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;oBACxC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;gBAC/C,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,GAAG,GAAG,CAAC,CAAC;YACV,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC/B,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;YAC3B,CAAC;YAED,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,wBAAwB,CAAC,CAAC;IAC5C,CAAC;IAED,SAAS,CAAC,GAAsB;QAC9B,MAAM,EAAC,EAAE,EAAE,SAAS,EAAC,GAAG,GAAG,CAAC;QAC5B,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAa;YACjD,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC;SAClD,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;QAC7D,IAAI,GAAG,CAAC,cAAc,KAAK,IAAI,CAAC,eAAe,EAAE,CAAC;YAChD,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,qBAAqB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;YAEvD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAChC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAa;QACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC5B,CAAC;CACF"}

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts

@@ -1,110 +0,0 @@
-import type { Source } from '../../types/streams.js';
-import type { Service } from '../service.js';
-import type { Change, DataChange, MessageBegin, MessageCommit } from './schema/change.js';
-/**
- * The ChangeStreamer is the component between replicators ("subscribers")
- * and a canonical upstream source of changes (e.g. a Postgres logical
- * replication slot). It facilitates multiple subscribers without incurring
- * the associated upstream expense (e.g. PG replication slots are resource
- * intensive) with a "forward-store-ack" procedure.
- *
- * * Changes from the upstream source are immediately **forwarded** to
- *   connected subscribers to minimize latency.
- *
- * * They are then **stored** in a separate DB to facilitate catchup
- *   of connecting subscribers that are behind.
- *
- * * **Acknowledgements** are sent upstream after they are successfully
- *   stored.
- *
- * **Cleanup** (Not yet implemented)
- *
- * Unlike Postgres replication slots, in which the progress of a static
- * subscriber is tracked in the replication slot, the ChangeStreamer
- * supports a dynamic set of subscribers (i.e.. zero-caches) that can
- * can continually change.
- *
- * However, it is not the case that the ChangeStreamer needs to support
- * arbitrarily old subscribers. Because the replica is continually
- * backed up to a global location and used to initialize new subscriber
- * tasks, an initial subscription request from a subscriber constitutes
- * a signal for how "behind" a new subscriber task can be. This is
- * reflected in the {@link SubscriberContext}, which indicates whether
- * the watermark corresponds to an "initial" watermark derived from the
- * replica at task startup.
- *
- * The ChangeStreamer uses a combination of this signal with ACK
- * responses from connected subscribers to determine the watermark up
- * to which it is safe to purge old change log entries.
- */
-export interface ChangeStreamer {
-    /**
-     * Subscribes to changes based on the supplied subscriber `ctx`,
-     * which indicates the watermark at which the subscriber is up to
-     * date.
-     */
-    subscribe(ctx: SubscriberContext): Source<Downstream>;
-}
-export type SubscriberContext = {
-    /**
-     * Subscriber id. This is only used for debugging.
-     */
-    id: string;
-    /**
-     * The ChangeStreamer will return an Error if the subscriber is
-     * on a different replica version (i.e. the initial snapshot associated
-     * with the replication slot).
-     */
-    replicaVersion: string;
-    /**
-     * The watermark up to which the subscriber is up to date.
-     * Only changes after the watermark will be streamed.
-     */
-    watermark: string;
-    /**
-     * Whether this is the first subscription request made by the task,
-     * i.e. indicating that the watermark comes from a restored replica
-     * backup. The ChangeStreamer uses this to determine which changes
-     * are safe to purge from the Storer.
-     */
-    initial: boolean;
-};
-export type ChangeEntry = {
-    change: Change;
-    /**
-     * Note that it is technically possible for multiple changes to have
-     * the same watermark, but that of a commit is guaranteed to be final,
-     * so subscribers should only store the watermark of commit changes.
-     */
-    watermark: string;
-};
-export declare const enum ErrorType {
-    Unknown = 0,
-    WrongReplicaVersion = 1,
-    WatermarkTooOld = 2
-}
-export type SubscriptionError = {
-    type: ErrorType;
-    message?: string | undefined;
-};
-export type Begin = ['begin', MessageBegin];
-export type Data = ['data', DataChange];
-export type Commit = ['commit', MessageCommit, {
-    watermark: string;
-}];
-export type Error = ['error', SubscriptionError];
-export type DownstreamChange = Begin | Data | Commit;
-/**
- * A stream of transactions, each starting with a {@link MessageBegin},
- * containing one or more {@link DataChange}s, and ending with a
- * {@link MessageCommit}. The 'commit' tuple includes a `watermark` that
- * should be stored with the committed data and used for resuming
- * a subscription (e.g. in the {@link SubscriberContext}).
- *
- * A {@link SubscriptionError} indicates an unrecoverable error that requires
- * manual intervention (e.g. configuration / operational error).
- */
-export type Downstream = DownstreamChange | Error;
-export interface ChangeStreamerService extends ChangeStreamer, Service {
-}
-//# sourceMappingURL=change-streamer.d.ts.map

package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EACV,MAAM,EACN,UAAU,EACV,YAAY,EACZ,aAAa,EACd,MAAM,oBAAoB,CAAC;AAE5B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IAEH,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;CACvD;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,0BAAkB,SAAS;IACzB,OAAO,IAAA;IACP,mBAAmB,IAAA;IACnB,eAAe,IAAA;CAChB;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,SAAS,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;AAC5C,MAAM,MAAM,IAAI,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;AACxC,MAAM,MAAM,MAAM,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE;IAAC,SAAS,EAAE,MAAM,CAAA;CAAC,CAAC,CAAC;AACpE,MAAM,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;AAEjD,MAAM,MAAM,gBAAgB,GAAG,KAAK,GAAG,IAAI,GAAG,MAAM,CAAC;AAErD;;;;;;;;;GASG;AACH,MAAM,MAAM,UAAU,GAAG,gBAAgB,GAAG,KAAK,CAAC;AAElD,MAAM,WAAW,qBAAsB,SAAQ,cAAc,EAAE,OAAO;CAAG"}

package/out/zero-cache/src/services/change-streamer/change-streamer.js

@@ -1,7 +0,0 @@
-export var ErrorType;
-(function (ErrorType) {
-    ErrorType[ErrorType["Unknown"] = 0] = "Unknown";
-    ErrorType[ErrorType["WrongReplicaVersion"] = 1] = "WrongReplicaVersion";
-    ErrorType[ErrorType["WatermarkTooOld"] = 2] = "WatermarkTooOld";
-})(ErrorType || (ErrorType = {}));
-//# sourceMappingURL=change-streamer.js.map

package/out/zero-cache/src/services/change-streamer/change-streamer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"change-streamer.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"names":[],"mappings":"AA8FA,MAAM,CAAN,IAAkB,SAIjB;AAJD,WAAkB,SAAS;IACzB,+CAAO,CAAA;IACP,uEAAmB,CAAA;IACnB,+DAAe,CAAA;AACjB,CAAC,EAJiB,SAAS,KAAT,SAAS,QAI1B"}

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts

@@ -1,19 +0,0 @@
-import type { WatermarkedChange } from './change-streamer-service.js';
-import { Subscriber } from './subscriber.js';
-export declare class Forwarder {
-    #private;
-    /**
-     * `add()` is called in lock step with `Storer.catchup()` so that the
-     * two components have an equivalent interpretation of whether a Transaction is
-     * currently being streamed.
-     */
-    add(sub: Subscriber): void;
-    remove(sub: Subscriber): void;
-    /**
-     * `forward()` is called in lockstep with `Storer.store()` so that the
-     * two components have an equivalent interpretation of whether a Transaction is
-     * currently being streamed.
-     */
-    forward(entry: WatermarkedChange): void;
-}
-//# sourceMappingURL=forwarder.d.ts.map

package/out/zero-cache/src/services/change-streamer/forwarder.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"forwarder.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAE3C,qBAAa,SAAS;;IAKpB;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,UAAU;IAQnB,MAAM,CAAC,GAAG,EAAE,UAAU;IAMtB;;;;OAIG;IACH,OAAO,CAAC,KAAK,EAAE,iBAAiB;CA0BjC"}

package/out/zero-cache/src/services/change-streamer/forwarder.js

@@ -1,56 +0,0 @@
-import { Subscriber } from './subscriber.js';
-export class Forwarder {
-    #active = new Set();
-    #queued = new Set();
-    #inTransaction = false;
-    /**
-     * `add()` is called in lock step with `Storer.catchup()` so that the
-     * two components have an equivalent interpretation of whether a Transaction is
-     * currently being streamed.
-     */
-    add(sub) {
-        if (this.#inTransaction) {
-            this.#queued.add(sub);
-        }
-        else {
-            this.#active.add(sub);
-        }
-    }
-    remove(sub) {
-        this.#active.delete(sub);
-        this.#queued.delete(sub);
-        sub.close();
-    }
-    /**
-     * `forward()` is called in lockstep with `Storer.store()` so that the
-     * two components have an equivalent interpretation of whether a Transaction is
-     * currently being streamed.
-     */
-    forward(entry) {
-        const [type] = entry[1];
-        for (const active of this.#active.values()) {
-            active.send(entry);
-        }
-        switch (type) {
-            case 'begin':
-                // While in a Transaction, all added subscribers are "queued" so that no
-                // messages are forwarded to them. This state corresponds to being queued
-                // for catchup in the Storer, which will retrieve historic changes
-                // and call catchup() once the current transaction is committed.
-                this.#inTransaction = true;
-                break;
-            case 'commit':
-                // Upon commit, all queued subscribers are transferred to the active set.
-                // This means that they can receive messages starting from the next transaction.
-                // Note that if catchup is still in progress (in the Storer), these messages
-                // will be buffered in the backlog until catchup completes.
-                this.#inTransaction = false;
-                for (const sub of this.#queued.values()) {
-                    this.#active.add(sub);
-                }
-                this.#queued.clear();
-                break;
-        }
-    }
-}
-//# sourceMappingURL=forwarder.js.map

package/out/zero-cache/src/services/change-streamer/forwarder.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"forwarder.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/forwarder.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAE3C,MAAM,OAAO,SAAS;IACX,OAAO,GAAG,IAAI,GAAG,EAAc,CAAC;IAChC,OAAO,GAAG,IAAI,GAAG,EAAc,CAAC;IACzC,cAAc,GAAG,KAAK,CAAC;IAEvB;;;;OAIG;IACH,GAAG,CAAC,GAAe;QACjB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,MAAM,CAAC,GAAe;QACpB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,GAAG,CAAC,KAAK,EAAE,CAAC;IACd,CAAC;IAED;;;;OAIG;IACH,OAAO,CAAC,KAAwB;QAC9B,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACxB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QACD,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,OAAO;gBACV,wEAAwE;gBACxE,yEAAyE;gBACzE,kEAAkE;gBAClE,gEAAgE;gBAChE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC3B,MAAM;YACR,KAAK,QAAQ;gBACX,yEAAyE;gBACzE,gFAAgF;gBAChF,4EAA4E;gBAC5E,2DAA2D;gBAC3D,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;gBAC5B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;oBACxC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACxB,CAAC;gBACD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACrB,MAAM;QACV,CAAC;IACH,CAAC;CACF"}

package/out/zero-cache/src/services/change-streamer/pg/change-source.d.ts

@@ -1,14 +0,0 @@
-import { LogContext } from '@rocicorp/logger';
-import type { ChangeSource } from '../change-streamer-service.js';
-import type { ReplicationConfig } from '../schema/tables.js';
-import type { ShardConfig } from './shard-config.js';
-/**
- * Initializes a Postgres change source, including the initial sync of the
- * replica, before streaming changes from the corresponding logical replication
- * stream.
- */
-export declare function initializeChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string): Promise<{
-    replicationConfig: ReplicationConfig;
-    changeSource: ChangeSource;
-}>;
-//# sourceMappingURL=change-source.d.ts.map

package/out/zero-cache/src/services/change-streamer/pg/change-source.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/pg/change-source.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAsB5C,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,+BAA+B,CAAC;AAG9E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,qBAAqB,CAAC;AAI3D,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,mBAAmB,CAAC;AAMnD;;;;GAIG;AACH,wBAAsB,sBAAsB,CAC1C,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAkC7E"}

package/out/zero-cache/src/services/change-streamer/pg/change-source.js

@@ -1,211 +0,0 @@
-import { PG_ADMIN_SHUTDOWN } from '@drdgvhbh/postgres-error-codes';
-import { LogContext } from '@rocicorp/logger';
-import { resolver } from '@rocicorp/resolver';
-import { LogicalReplicationService, Pgoutput, PgoutputPlugin, } from 'pg-logical-replication';
-import { DatabaseError } from 'pg-protocol';
-import { AbortError } from '../../../../../shared/src/abort-error.js';
-import { deepEqual } from '../../../../../shared/src/json.js';
-import { sleep } from '../../../../../shared/src/sleep.js';
-import { StatementRunner } from '../../../db/statements.js';
-import { stringify } from '../../../types/bigint-json.js';
-import { max, oneAfter } from '../../../types/lexi-version.js';
-import { pgClient, registerPostgresTypeParsers, } from '../../../types/pg.js';
-import { Subscription } from '../../../types/subscription.js';
-import { Database } from '../../../../../zqlite/src/db.js';
-import { getSubscriptionState } from '../../replicator/schema/replication-state.js';
-import { replicationSlot } from './initial-sync.js';
-import { fromLexiVersion, toLexiVersion } from './lsn.js';
-import { INTERNAL_PUBLICATION_PREFIX } from './schema/zero.js';
-import { initSyncSchema } from './sync-schema.js';
-// BigInt support from LogicalReplicationService.
-registerPostgresTypeParsers();
-/**
- * Initializes a Postgres change source, including the initial sync of the
- * replica, before streaming changes from the corresponding logical replication
- * stream.
- */
-export async function initializeChangeSource(lc, upstreamURI, shard, replicaDbFile) {
-    await initSyncSchema(lc, 'change-streamer', shard, replicaDbFile, upstreamURI);
-    const replica = new Database(lc, replicaDbFile);
-    const replicationConfig = getSubscriptionState(new StatementRunner(replica));
-    replica.close();
-    if (shard.publications.length) {
-        // Verify that the publications match what has been synced.
-        const requested = [...shard.publications].sort();
-        const replicated = replicationConfig.publications
-            .filter(p => !p.startsWith(INTERNAL_PUBLICATION_PREFIX))
-            .sort();
-        if (!deepEqual(requested, replicated)) {
-            throw new Error(`Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`);
-        }
-    }
-    const changeSource = new PostgresChangeSource(lc, upstreamURI, shard.id, replicationConfig);
-    return { replicationConfig, changeSource };
-}
-/**
- * Postgres implementation of a {@link ChangeSource} backed by a logical
- * replication stream.
- */
-class PostgresChangeSource {
-    #lc;
-    #upstreamUri;
-    #shardID;
-    #replicationConfig;
-    constructor(lc, upstreamUri, shardID, replicationConfig) {
-        this.#lc = lc.withContext('component', 'change-source');
-        this.#upstreamUri = upstreamUri;
-        this.#shardID = shardID;
-        this.#replicationConfig = replicationConfig;
-    }
-    async startStream(clientWatermark) {
-        const db = pgClient(this.#lc, this.#upstreamUri);
-        const slot = replicationSlot(this.#shardID);
-        const clientStart = oneAfter(clientWatermark);
-        try {
-            let lastLSN = '0/0';
-            const ack = (commit) => {
-                if (commit) {
-                    const { watermark } = commit[2];
-                    lastLSN = fromLexiVersion(watermark);
-                }
-                void service.acknowledge(lastLSN);
-            };
-            const changes = Subscription.create({
-                cleanup: () => service.stop(),
-            });
-            // To avoid a race condition when handing off the replication stream
-            // between tasks, query the `confirmed_flush_lsn` for the replication
-            // slot only after the replication stream starts, as that is when it
-            // is guaranteed not to change (i.e. until we ACK a commit).
-            const { promise: nextWatermark, resolve, reject } = resolver();
-            const service = new LogicalReplicationService({ connectionString: this.#upstreamUri }, { acknowledge: { auto: false, timeoutSeconds: 0 } })
-                .on('start', () => this.getNextWatermark(db, slot, clientStart)
-                .then(resolve)
-                .catch(e => reject(translateError(e))))
-                .on('heartbeat', (_lsn, _time, respond) => {
-                respond && ack();
-            })
-                .on('data', (lsn, msg) => {
-                const change = messageToDownstream(lsn, msg);
-                if (change) {
-                    changes.push(change);
-                }
-            })
-                .on('error', err => {
-                this.#lc.error?.('error from upstream postgres', err);
-                changes.fail(err);
-            });
-            await this.stopExistingReplicationSlotSubscriber(db, slot);
-            this.#lc.info?.(`starting replication stream @${slot}`);
-            service
-                .subscribe(new PgoutputPlugin({
-                protoVersion: 1,
-                publicationNames: this.#replicationConfig.publications,
-            }), slot, fromLexiVersion(clientStart))
-                .catch(e => {
-                const err = translateError(e);
-                reject(err);
-                changes.fail(err);
-            })
-                .finally(() => changes.cancel());
-            const initialWatermark = await nextWatermark;
-            this.#lc.info?.(`replication stream@${slot} started at ${initialWatermark}`);
-            return { initialWatermark, changes, acks: { push: ack } };
-        }
-        finally {
-            await db.end();
-        }
-    }
-    async stopExistingReplicationSlotSubscriber(db, slot) {
-        const result = await db `
-    SELECT pg_terminate_backend(active_pid), active_pid as pid
-      FROM pg_replication_slots WHERE slot_name = ${slot} and active = true`;
-        if (result.length === 0) {
-            this.#lc.debug?.(`no existing subscriber to replication slot`);
-        }
-        else {
-            const { pid } = result[0];
-            this.#lc.info?.(`signaled subscriber ${pid} to shut down`);
-            // This reduces flakiness in which unit tests often fail with
-            // an error when starting the replication stream:
-            //
-            // error: replication slot "zero_slot_change_source_test_id" is active for PID 268
-            //
-            // Presumably, waiting for small interval before connecting to Postgres
-            // would also reduce this occurrence in production.
-            await sleep(5);
-        }
-    }
-    // Sometimes the `confirmed_flush_lsn` gets wiped, e.g.
-    //
-    // ```
-    //  slot_name | restart_lsn | confirmed_flush_lsn
-    //  -----------+-------------+---------------------
-    //  zero_slot | 8F/38ACB2F8 | 0/1
-    // ```
-    //
-    // Using the greatest of three values should always yield a correct result:
-    // * `clientWatermark`    : ahead of `confirmed_flush_lsn` if an ACK was lost,
-    //                          or if the `confirmed_flush_lsn` was wiped.
-    // * `confirmed_flush_lsn`: ahead of the `clientWatermark` if the ChangeDB was wiped.
-    // * `restart_lsn`        : if both the `confirmed_flush_lsn` and ChangeDB were wiped.
-    async getNextWatermark(db, slot, clientStart) {
-        const result = await db `
-      SELECT restart_lsn as restart, confirmed_flush_lsn as confirmed FROM pg_replication_slots 
-        WHERE slot_name = ${slot}`;
-        if (result.length === 0) {
-            throw new Error(`Upstream is missing replication slot ${slot}`);
-        }
-        const { restart, confirmed } = result[0];
-        const confirmedWatermark = toLexiVersion(confirmed);
-        const restartWatermark = toLexiVersion(restart);
-        this.#lc.info?.(`confirmed_flush_lsn:${confirmed}, restart_lsn:${restart}, clientWatermark:${fromLexiVersion(clientStart)}`);
-        return max(oneAfter(confirmedWatermark), oneAfter(restartWatermark), clientStart);
-    }
-}
-function messageToDownstream(lsn, msg) {
-    const change = msg;
-    const { tag } = change;
-    switch (tag) {
-        case 'begin':
-            return ['begin', change];
-        case 'insert':
-        case 'update':
-        case 'delete':
-        case 'truncate':
-            return ['data', change];
-        case 'commit': {
-            const watermark = toLexiVersion(lsn);
-            return ['commit', change, { watermark }];
-        }
-        default:
-            change; // All Change types are covered.
-            // But we can technically receive other Message types.
-            switch (msg.tag) {
-                case 'relation':
-                    return undefined; // Explicitly ignored. Schema handling is TODO.
-                case 'type':
-                    throw new Error(`Custom types are not supported (received "${msg.typeName}")`);
-                case 'origin':
-                    // We do not set the `origin` option in the pgoutput parameters:
-                    // https://www.postgresql.org/docs/current/protocol-logical-replication.html#PROTOCOL-LOGICAL-REPLICATION-PARAMS
-                    throw new Error(`Unexpected ORIGIN message ${stringify(msg)}`);
-                case 'message':
-                    // We do not set the `messages` option in the pgoutput parameters:
-                    // https://www.postgresql.org/docs/current/protocol-logical-replication.html#PROTOCOL-LOGICAL-REPLICATION-PARAMS
-                    throw new Error(`Unexpected MESSAGE message ${stringify(msg)}`);
-                default:
-                    throw new Error(`Unexpected message type ${stringify(msg)}`);
-            }
-    }
-}
-function translateError(e) {
-    if (!(e instanceof Error)) {
-        return new Error(String(e));
-    }
-    if (e instanceof DatabaseError && e.code === PG_ADMIN_SHUTDOWN) {
-        return new AbortError(e.message, { cause: e });
-    }
-    return e;
-}
-//# sourceMappingURL=change-source.js.map