@rocicorp/zero 0.0.0-202410031711

package/out/zero-cache/src/services/replicator/checkpointer.d.ts

@@ -0,0 +1,79 @@
+import { LogContext } from '@rocicorp/logger';
+import { Notifier } from './notifier.js';
+/**
+ * A `Checkpointer` is consulted by the Replicator after each commit and given
+ * the opportunity to perform a blocking WAL checkpoint before the Replicator
+ * continues processing the replication stream.
+ */
+export interface Checkpointer {
+    maybeCheckpoint(numCommittedChanges: number, notifier: Notifier): Promise<void>;
+    stop(): void;
+}
+/**
+ * The `NULL_CHECKPOINTER` is suitable for an environment in which checkpoints are
+ * handled by an external entity, such as a `litestream replicate` process.
+ */
+export declare const NULL_CHECKPOINTER: Checkpointer;
+export type CheckpointerConfig = {
+    /**
+     * The number of outstanding frames or changes at which a blocking
+     * checkpoint is triggered.
+     *
+     * Defaults to 200.
+     */
+    threshold?: number;
+    /**
+     * The base timeout to block on an active checkpoint. This timeout is increased
+     * in proportion to the size of WAL divided by the `threshold`. For example, if
+     * the size of the WAL is twice the `threshold`, the timeout will be doubled.
+     *
+     * This dynamic timeout algorithm reduces the amount of system-wide pausing in
+     * the common case, even in the presence of occasional, long-held locks due to
+     * events like a long hydration. If checkpoint is not able to start within the
+     * timeout, the checkpointer gives up and allows the system to progress. As the
+     * size of the WAL grows, the checkpointer will wait for longer timeouts to ensure
+     * that the checkpointing eventually succeeds.
+     *
+     * Defaults to 200 milliseconds.
+     */
+    baseCheckpointTimeoutMs?: number;
+    /**
+     * A regular interval at which passive checkpoints are attempted. This allows an
+     * idle task (i.e. no sync connections) to clean up outside of the
+     * replication path.
+     *
+     * Defaults to one minute.
+     */
+    passiveCheckpointPeriodMs?: number;
+};
+/**
+ * The `WALCheckpointer` executes checkpoints when the number of WAL log entries
+ * exceeds a configurable threshold. It does this by:
+ *
+ * 1. Broadcasting a `maintenance` ReplicaState message to signal view-syncers to
+ *    release their locks.
+ * 2. Executing `wal_checkpoint(TRUNCATE)` with a busy_timeout proportional to
+ *    the outstanding log size.
+ * 3. Broadcasting a `version-ready` ReplicaState message to signal view-syncers to
+ *    reestablish their locks.
+ *
+ * Points of interest:
+ *
+ * * The checkpointer waits for the `maintenance` broadcast to be sent to over
+ *   IPC channels (in-process blocking), but does not request or wait for ACKs from
+ *   the view syncers. This is because the `wal_checkpoint(TRUNCATE)` command itself
+ *   will block until all read locks are released, up to the configured `busy_timeout`.
+ *
+ * * After performing the checkpoint, however, the checkpointer _does_ request ACKs
+ *   from view-syncers to confirm that they have reestablished their snapshots. There
+ *   is otherwise no way to know when it is safe to continue replication. This is
+ *   expected to be very fast, but to avoid pathological cases the wait is capped at
+ *   100ms.
+ */
+export declare class WALCheckpointer implements Checkpointer {
+    #private;
+    constructor(lc: LogContext, replicaDbFile: string, cfg?: CheckpointerConfig);
+    stop(): void;
+    maybeCheckpoint(changes: number, notifier: Notifier): Promise<void>;
+}
+//# sourceMappingURL=checkpointer.d.ts.map

package/out/zero-cache/src/services/replicator/checkpointer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpointer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/checkpointer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAM5C,OAAO,EAAC,QAAQ,EAAC,MAAM,eAAe,CAAC;AAEvC;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,eAAe,CACb,mBAAmB,EAAE,MAAM,EAC3B,QAAQ,EAAE,QAAQ,GACjB,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB,IAAI,IAAI,IAAI,CAAC;CACd;AAED;;;GAGG;AACH,eAAO,MAAM,iBAAiB,EAAE,YAGtB,CAAC;AAEX,MAAM,MAAM,kBAAkB,GAAG;IAC/B;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;;;;;;;;;;;OAaG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;OAMG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAC;CACpC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,qBAAa,eAAgB,YAAW,YAAY;;gBAWhD,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM,EACrB,GAAG,GAAE,kBAAuB;IA4B9B,IAAI;IAKE,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ;CAwE1D"}

package/out/zero-cache/src/services/replicator/checkpointer.js

@@ -0,0 +1,124 @@
+import { LogContext } from '@rocicorp/logger';
+import { assert } from '../../../../shared/src/asserts.js';
+import { randInt } from '../../../../shared/src/rand.js';
+import { promiseVoid } from '../../../../shared/src/resolved-promises.js';
+import { orTimeout } from '../../../../zero-cache/src/types/timeout.js';
+import { Database } from '../../../../zqlite/src/db.js';
+import { Notifier } from './notifier.js';
+/**
+ * The `NULL_CHECKPOINTER` is suitable for an environment in which checkpoints are
+ * handled by an external entity, such as a `litestream replicate` process.
+ */
+export const NULL_CHECKPOINTER = {
+    maybeCheckpoint: () => promiseVoid,
+    stop: () => { },
+};
+/**
+ * The `WALCheckpointer` executes checkpoints when the number of WAL log entries
+ * exceeds a configurable threshold. It does this by:
+ *
+ * 1. Broadcasting a `maintenance` ReplicaState message to signal view-syncers to
+ *    release their locks.
+ * 2. Executing `wal_checkpoint(TRUNCATE)` with a busy_timeout proportional to
+ *    the outstanding log size.
+ * 3. Broadcasting a `version-ready` ReplicaState message to signal view-syncers to
+ *    reestablish their locks.
+ *
+ * Points of interest:
+ *
+ * * The checkpointer waits for the `maintenance` broadcast to be sent to over
+ *   IPC channels (in-process blocking), but does not request or wait for ACKs from
+ *   the view syncers. This is because the `wal_checkpoint(TRUNCATE)` command itself
+ *   will block until all read locks are released, up to the configured `busy_timeout`.
+ *
+ * * After performing the checkpoint, however, the checkpointer _does_ request ACKs
+ *   from view-syncers to confirm that they have reestablished their snapshots. There
+ *   is otherwise no way to know when it is safe to continue replication. This is
+ *   expected to be very fast, but to avoid pathological cases the wait is capped at
+ *   100ms.
+ */
+export class WALCheckpointer {
+    #lc;
+    #db;
+    #threshold;
+    #baseCheckpointTimeoutMs;
+    #passiveCheckpointTimer;
+    #logSize = 0;
+    #outstandingChanges = 0;
+    constructor(lc, replicaDbFile, cfg = {}) {
+        const { threshold = 200, baseCheckpointTimeoutMs = 10, passiveCheckpointPeriodMs = 60_000, } = cfg;
+        assert(threshold > 0 &&
+            baseCheckpointTimeoutMs > 0 &&
+            passiveCheckpointPeriodMs > 0, `Invalid config ${JSON.stringify(cfg)}`);
+        const db = new Database(lc, replicaDbFile);
+        db.pragma('journal_mode = WAL');
+        this.#lc = lc.withContext('component', 'wal-checkpointer');
+        this.#db = db;
+        this.#threshold = threshold;
+        this.#baseCheckpointTimeoutMs = baseCheckpointTimeoutMs;
+        this.#passiveCheckpointTimer = setInterval(() => this.#checkpoint('PASSIVE'), passiveCheckpointPeriodMs);
+    }
+    stop() {
+        clearTimeout(this.#passiveCheckpointTimer);
+        this.#lc.info?.('stopped');
+    }
+    async maybeCheckpoint(changes, notifier) {
+        this.#outstandingChanges += changes;
+        // Simplification: changes and frames equivalently w.r.t. the threshold.
+        if (Math.max(this.#outstandingChanges, this.#logSize) >= this.#threshold) {
+            // If no read locks are held, a PASSIVE checkpoint may suffice.
+            // Regardless of success, #logSize gets updated to determine if a
+            // blocking checkpoint is warranted.
+            this.#checkpoint('PASSIVE');
+        }
+        if (this.#logSize >= this.#threshold) {
+            const logSize = this.#logSize;
+            const t0 = Date.now();
+            await this.#enterMaintenanceMode(notifier);
+            const t1 = Date.now();
+            // The timeout is proportional to the size of the log compared to the threshold.
+            const timeout = (this.#logSize / this.#threshold) * this.#baseCheckpointTimeoutMs;
+            const result = this.#checkpoint('TRUNCATE', timeout);
+            const t2 = Date.now();
+            await this.#exitMaintenanceMode(notifier);
+            const t3 = Date.now();
+            this.#lc.info?.(`WAL(busy=${timeout}ms): pre=${t1 - t0}ms checkpoint=${t2 - t1}ms post=${t3 - t2}ms logSize=${logSize} result`, result);
+        }
+    }
+    async #enterMaintenanceMode(notifier) {
+        await Promise.all(notifier.notifySubscribers({ state: 'maintenance' }));
+    }
+    async #exitMaintenanceMode(notifier) {
+        // Request an ACK when exiting maintenance mode to maximize the chance of
+        // view syncers re-establishing snapshots at the same version. This is expected
+        // to be very fast (single-digit milliseconds).
+        //
+        // However, this is also the chance for view syncers to invoke an occasional
+        // `PRAGMA optimize` call, for which latency may be variable. To avoid holding
+        // up the replication stream indefinitely, we cap the wait at 100ms.
+        const result = await orTimeout(Promise.all(notifier.notifySubscribers({
+            state: 'version-ready',
+            ack: randomACK(),
+        })), 100);
+        if (result === 'timed-out') {
+            this.#lc.info?.('timed out waiting for view-syncer resumption');
+        }
+    }
+    #checkpoint(mode, timeoutMs) {
+        if (timeoutMs) {
+            this.#db.pragma(`busy_timeout = ${timeoutMs}`);
+        }
+        const result = checkpoint(this.#db, mode);
+        this.#logSize = result.log;
+        this.#outstandingChanges = 0;
+        return result;
+    }
+}
+function checkpoint(db, mode) {
+    const result = db.pragma(`wal_checkpoint(${mode})`);
+    return result[0];
+}
+function randomACK() {
+    return randInt(Number.MIN_SAFE_INTEGER, Number.MAX_SAFE_INTEGER);
+}
+//# sourceMappingURL=checkpointer.js.map

package/out/zero-cache/src/services/replicator/checkpointer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpointer.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/checkpointer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,MAAM,EAAC,MAAM,uBAAuB,CAAC;AAC7C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAC,WAAW,EAAC,MAAM,iCAAiC,CAAC;AAC5D,OAAO,EAAC,SAAS,EAAC,MAAM,iCAAiC,CAAC;AAC1D,OAAO,EAAC,QAAQ,EAAC,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAC,QAAQ,EAAC,MAAM,eAAe,CAAC;AAgBvC;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAiB;IAC7C,eAAe,EAAE,GAAG,EAAE,CAAC,WAAW;IAClC,IAAI,EAAE,GAAG,EAAE,GAAE,CAAC;CACN,CAAC;AAqCX;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,OAAO,eAAe;IACjB,GAAG,CAAa;IAChB,GAAG,CAAW;IACd,UAAU,CAAS;IACnB,wBAAwB,CAAS;IACjC,uBAAuB,CAAgC;IAEhE,QAAQ,GAAG,CAAC,CAAC;IACb,mBAAmB,GAAG,CAAC,CAAC;IAExB,YACE,EAAc,EACd,aAAqB,EACrB,MAA0B,EAAE;QAE5B,MAAM,EACJ,SAAS,GAAG,GAAG,EACf,uBAAuB,GAAG,EAAE,EAC5B,yBAAyB,GAAG,MAAM,GACnC,GAAG,GAAG,CAAC;QAER,MAAM,CACJ,SAAS,GAAG,CAAC;YACX,uBAAuB,GAAG,CAAC;YAC3B,yBAAyB,GAAG,CAAC,EAC/B,kBAAkB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CACxC,CAAC;QAEF,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,EAAE,EAAE,aAAa,CAAC,CAAC;QAC3C,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAEhC,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC,WAAW,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QAC3D,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,wBAAwB,GAAG,uBAAuB,CAAC;QACxD,IAAI,CAAC,uBAAuB,GAAG,WAAW,CACxC,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,EACjC,yBAAyB,CAC1B,CAAC;IACJ,CAAC;IAED,IAAI;QACF,YAAY,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAC3C,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe,EAAE,QAAkB;QACvD,IAAI,CAAC,mBAAmB,IAAI,OAAO,CAAC;QAEpC,wEAAwE;QACxE,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACzE,+DAA+D;YAC/D,iEAAiE;YACjE,oCAAoC;YACpC,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAC9B,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACrC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC;YAE9B,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;YAC3C,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEtB,gFAAgF;YAChF,MAAM,OAAO,GACX,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,wBAAwB,CAAC;YACpE,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YAErD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;YAC1C,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEtB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CACb,YAAY,OAAO,YAAY,EAAE,GAAG,EAAE,iBACpC,EAAE,GAAG,EACP,WAAW,EAAE,GAAG,EAAE,cAAc,OAAO,SAAS,EAChD,MAAM,CACP,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,QAAkB;QAC5C,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAC,KAAK,EAAE,aAAa,EAAC,CAAC,CAAC,CAAC;IACxE,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,QAAkB;QAC3C,yEAAyE;QACzE,+EAA+E;QAC/E,+CAA+C;QAC/C,EAAE;QACF,4EAA4E;QAC5E,8EAA8E;QAC9E,oEAAoE;QACpE,MAAM,MAAM,GAAG,MAAM,SAAS,CAC5B,OAAO,CAAC,GAAG,CACT,QAAQ,CAAC,iBAAiB,CAAC;YACzB,KAAK,EAAE,eAAe;YACtB,GAAG,EAAE,SAAS,EAAE;SACjB,CAAC,CACH,EACD,GAAG,CACJ,CAAC;QACF,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;YAC3B,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAED,WAAW,CAAC,IAAoB,EAAE,SAAkB;QAClD,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,kBAAkB,SAAS,EAAE,CAAC,CAAC;QACjD,CAAC;QACD,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAE1C,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC;QAC3B,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAUD,SAAS,UAAU,CAAC,EAAY,EAAE,IAAoB;IACpD,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB,IAAI,GAAG,CAAuB,CAAC;IAC1E,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC;AAED,SAAS,SAAS;IAChB,OAAO,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC;AACnE,CAAC"}

package/out/zero-cache/src/services/replicator/incremental-sync.d.ts

@@ -0,0 +1,39 @@
+import type { LogContext } from '@rocicorp/logger';
+import { StatementRunner } from '../../../../zero-cache/src/db/statements.js';
+import { Database } from '../../../../zqlite/src/db.js';
+import type { Source } from '../../types/streams.js';
+import type { ChangeStreamer, DownstreamChange } from '../change-streamer/change-streamer.js';
+import type { Checkpointer } from './checkpointer.js';
+import type { ReplicaState } from './replicator.js';
+/**
+ * The {@link IncrementalSyncer} manages a logical replication stream from upstream,
+ * handling application lifecycle events (start, stop) and retrying the
+ * connection with exponential backoff. The actual handling of the logical
+ * replication messages is done by the {@link MessageProcessor}.
+ */
+export declare class IncrementalSyncer {
+    #private;
+    constructor(id: string, changeStreamer: ChangeStreamer, replica: Database, checkpointer: Checkpointer);
+    run(lc: LogContext): Promise<void>;
+    subscribe(): Source<ReplicaState>;
+    stop(lc: LogContext, err?: unknown): Promise<void>;
+}
+/**
+ * The {@link MessageProcessor} partitions the stream of messages into transactions
+ * by creating a {@link TransactionProcessor} when a transaction begins, and dispatching
+ * messages to it until the commit is received.
+ *
+ * From https://www.postgresql.org/docs/current/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW :
+ *
+ * "The logical replication protocol sends individual transactions one by one.
+ *  This means that all messages between a pair of Begin and Commit messages
+ *  belong to the same transaction."
+ */
+export declare class MessageProcessor {
+    #private;
+    constructor(db: StatementRunner, acknowledge: (watermark: string) => unknown, failService: (lc: LogContext, err: unknown) => void);
+    abort(lc: LogContext, err?: unknown): void;
+    /** @return The number of changes committed. */
+    processMessage(lc: LogContext, downstream: DownstreamChange): number;
+}
+//# sourceMappingURL=incremental-sync.d.ts.map

package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EAAC,eAAe,EAAC,MAAM,iCAAiC,CAAC;AAEhE,OAAO,EAAC,QAAQ,EAAC,MAAM,kBAAkB,CAAC;AAI1C,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EACV,cAAc,EACd,gBAAgB,EACjB,MAAM,uCAAuC,CAAC;AAU/C,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAEpD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,iBAAiB,CAAC;AASlD;;;;;GAKG;AACH,qBAAa,iBAAiB;;gBAW1B,EAAE,EAAE,MAAM,EACV,cAAc,EAAE,cAAc,EAC9B,OAAO,EAAE,QAAQ,EACjB,YAAY,EAAE,YAAY;IAStB,GAAG,CAAC,EAAE,EAAE,UAAU;IAmDxB,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC;IAI3B,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO;CAKzC;AAoBD;;;;;;;;;;GAUG;AAEH,qBAAa,gBAAgB;;gBAWzB,EAAE,EAAE,eAAe,EACnB,WAAW,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,EAC3C,WAAW,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,KAAK,IAAI;IAwBrD,KAAK,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO;IAInC,+CAA+C;IAC/C,cAAc,CAAC,EAAE,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,GAAG,MAAM;CA4ErE"}

package/out/zero-cache/src/services/replicator/incremental-sync.js

@@ -0,0 +1,342 @@
+import { ident } from 'pg-format';
+import { LogicalReplicationService } from 'pg-logical-replication';
+import { AbortError } from '../../../../shared/src/abort-error.js';
+import { assert, unreachable } from '../../../../shared/src/asserts.js';
+import { StatementRunner } from '../../../../zero-cache/src/db/statements.js';
+import { liteValues } from '../../../../zero-cache/src/types/lite.js';
+import { Database } from '../../../../zqlite/src/db.js';
+import { stringify } from '../../types/bigint-json.js';
+import { liteTableName } from '../../types/names.js';
+import { RunningState } from '../running-state.js';
+import { Notifier } from './notifier.js';
+import { logDeleteOp, logSetOp, logTruncateOp } from './schema/change-log.js';
+import { ZERO_VERSION_COLUMN_NAME, getReplicationVersions, getSubscriptionState, updateReplicationWatermark, } from './schema/replication-state.js';
+/**
+ * The {@link IncrementalSyncer} manages a logical replication stream from upstream,
+ * handling application lifecycle events (start, stop) and retrying the
+ * connection with exponential backoff. The actual handling of the logical
+ * replication messages is done by the {@link MessageProcessor}.
+ */
+export class IncrementalSyncer {
+    #id;
+    #changeStreamer;
+    #replica;
+    #notifier;
+    #checkpointer;
+    #state = new RunningState('IncrementalSyncer');
+    #service;
+    constructor(id, changeStreamer, replica, checkpointer) {
+        this.#id = id;
+        this.#changeStreamer = changeStreamer;
+        this.#replica = new StatementRunner(replica);
+        this.#notifier = new Notifier();
+        this.#checkpointer = checkpointer;
+    }
+    async run(lc) {
+        lc.info?.(`Starting IncrementalSyncer`);
+        const { watermark: initialWatermark } = getSubscriptionState(this.#replica);
+        // Notify any waiting subscribers that the replica is ready to be read.
+        this.#notifier.notifySubscribers();
+        while (this.#state.shouldRun()) {
+            const { replicaVersion, watermark } = getSubscriptionState(this.#replica);
+            const downstream = this.#changeStreamer.subscribe({
+                id: this.#id,
+                watermark,
+                replicaVersion,
+                initial: watermark === initialWatermark,
+            });
+            const processor = new MessageProcessor(this.#replica, (_watermark) => { }, // TODO: Add ACKs to ChangeStreamer API
+            (lc, err) => this.stop(lc, err));
+            const unregister = this.#state.cancelOnStop(downstream);
+            try {
+                for await (const message of downstream) {
+                    if (message[0] === 'error') {
+                        // Unrecoverable error. Stop the service.
+                        await this.stop(lc, message[1]);
+                        break;
+                    }
+                    this.#state.resetBackoff();
+                    const committed = processor.processMessage(lc, message);
+                    if (committed > 0) {
+                        this.#notifier.notifySubscribers({ state: 'version-ready' });
+                        await this.#checkpointer.maybeCheckpoint(committed, this.#notifier);
+                    }
+                }
+                processor.abort(lc);
+            }
+            catch (e) {
+                lc.error?.('Received error from ChangeStreamer', e);
+                processor.abort(lc, e);
+            }
+            finally {
+                downstream.cancel();
+                unregister();
+            }
+            await this.#state.backoff(lc);
+        }
+        lc.info?.('IncrementalSyncer stopped');
+    }
+    subscribe() {
+        return this.#notifier.subscribe();
+    }
+    async stop(lc, err) {
+        this.#state.stop(lc, err);
+        this.#checkpointer.stop();
+        await this.#service?.stop();
+    }
+}
+function ensureError(err) {
+    if (err instanceof Error) {
+        return err;
+    }
+    const error = new Error();
+    error.cause = err;
+    return error;
+}
+class ReplayedTransactionError extends Error {
+    watermark;
+    constructor(watermark, commit) {
+        super(`${watermark} has already been processed: ${stringify(commit)}`);
+        this.watermark = watermark;
+    }
+}
+/**
+ * The {@link MessageProcessor} partitions the stream of messages into transactions
+ * by creating a {@link TransactionProcessor} when a transaction begins, and dispatching
+ * messages to it until the commit is received.
+ *
+ * From https://www.postgresql.org/docs/current/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW :
+ *
+ * "The logical replication protocol sends individual transactions one by one.
+ *  This means that all messages between a pair of Begin and Commit messages
+ *  belong to the same transaction."
+ */
+// Exported for testing.
+export class MessageProcessor {
+    #db;
+    #acknowledge;
+    #failService;
+    #currentTx = null;
+    #pendingChanges = 0;
+    #failure;
+    constructor(db, acknowledge, failService) {
+        this.#db = db;
+        this.#acknowledge = acknowledge;
+        this.#failService = failService;
+    }
+    #fail(lc, err) {
+        if (!this.#failure) {
+            this.#currentTx?.abort(lc); // roll back any pending transaction.
+            this.#failure = ensureError(err);
+            if (err instanceof AbortError) {
+                // Aborted by the service.
+                lc.info?.('stopping MessageProcessor');
+            }
+            else {
+                // Propagate the failure up to the service.
+                lc.error?.('Message Processing failed:', this.#failure);
+                this.#failService(lc, this.#failure);
+            }
+        }
+    }
+    abort(lc, err) {
+        this.#fail(lc, err ?? new AbortError());
+    }
+    /** @return The number of changes committed. */
+    processMessage(lc, downstream) {
+        const [type, message] = downstream;
+        if (this.#failure) {
+            lc.debug?.(`Dropping ${message.tag}`);
+            return 0;
+        }
+        try {
+            const watermark = type === 'commit' ? downstream[2].watermark : undefined;
+            return this.#processMessage(lc, message, watermark);
+        }
+        catch (e) {
+            if (e instanceof ReplayedTransactionError) {
+                lc.info?.(e);
+                this.#acknowledge(e.watermark);
+            }
+            else {
+                this.#fail(lc, e);
+            }
+        }
+        return 0;
+    }
+    /** @return The number of changes committed. */
+    #processMessage(lc, msg, watermark) {
+        if (msg.tag === 'begin') {
+            if (this.#currentTx) {
+                throw new Error(`Already in a transaction ${stringify(msg)}`);
+            }
+            this.#currentTx = new TransactionProcessor(this.#db);
+            return (this.#pendingChanges = 0);
+        }
+        // For non-begin messages, there should be a #currentTx set.
+        const tx = this.#currentTx;
+        if (!tx) {
+            throw new Error(`Received message outside of transaction: ${stringify(msg)}`);
+        }
+        if (msg.tag === 'commit') {
+            // Undef this.#currentTx to allow the assembly of the next transaction.
+            this.#currentTx = null;
+            assert(watermark);
+            const elapsedMs = tx.processCommit(msg, watermark);
+            lc.debug?.(`Committed tx (${elapsedMs} ms)`);
+            this.#acknowledge(watermark);
+            return this.#pendingChanges;
+        }
+        this.#pendingChanges++;
+        switch (msg.tag) {
+            case 'insert':
+                tx.processInsert(msg);
+                break;
+            case 'update':
+                tx.processUpdate(msg);
+                break;
+            case 'delete':
+                tx.processDelete(msg);
+                break;
+            case 'truncate':
+                tx.processTruncate(msg);
+                break;
+            default:
+                unreachable(msg);
+        }
+        return 0;
+    }
+}
+/**
+ * The {@link TransactionProcessor} handles the sequence of messages from
+ * upstream, from `BEGIN` to `COMMIT` and executes the corresponding mutations
+ * on the {@link postgres.TransactionSql} on the replica.
+ *
+ * When applying row contents to the replica, the `_0_version` column is added / updated,
+ * and a corresponding entry in the `ChangeLog` is added. The version value is derived
+ * from the watermark of the preceding transaction (stored as the `nextStateVersion` in the
+ * `ReplicationState` table).
+ *
+ *   Side note: For non-streaming Postgres transactions, the commitEndLsn (and thus
+ *   commit watermark) is available in the `begin` message, so it could theoretically
+ *   be used for the row version of changes within the transaction. However, the
+ *   commitEndLsn is not available in the streaming (in-progress) transaction
+ *   protocol, and may not be available for CDC streams of other upstream types.
+ *   Therefore, the zero replication protocol is designed to not require the commit
+ *   watermark when a transaction begins.
+ *
+ * Also of interest is the fact that all INSERT Messages are logically applied as
+ * UPSERTs. See {@link processInsert} for the underlying motivation.
+ */
+class TransactionProcessor {
+    #startMs;
+    #db;
+    #version;
+    constructor(db) {
+        this.#startMs = Date.now();
+        // Although the Replicator / Incremental Syncer is the only writer of the replica,
+        // a `BEGIN CONCURRENT` transaction is used to allow View Syncers to simulate
+        // (i.e. and `ROLLBACK`) changes on historic snapshots of the database for the
+        // purpose of IVM).
+        //
+        // This TransactionProcessor is the only logic that will actually
+        // `COMMIT` any transactions to the replica.
+        db.beginConcurrent();
+        const { nextStateVersion } = getReplicationVersions(db);
+        this.#db = db;
+        this.#version = nextStateVersion;
+    }
+    /**
+     * Note: All INSERTs are processed a UPSERTs in order to properly handle
+     * replayed transactions (e.g. if an acknowledgement was lost). In the case
+     * of a replayed transaction, the final commit results in an rollback if the
+     * watermark is earlier than what has already been processed.
+     * See {@link processCommit}.
+     *
+     * Note that a transaction replay could theoretically be detected at the BEGIN message
+     * since it contains the commitEndLsn (from which a watermark can be derived), but
+     * that would not generalize to streaming transactions for which the commitEndLsn
+     * is not known until STREAM COMMIT.
+     *
+     * This UPSERT strategy instead handles both protocols by accepting all messages and
+     * making the COMMIT/ROLLBACK decision when the commit watermark is guaranteed to be
+     * available.
+     */
+    processInsert(insert) {
+        const table = liteTableName(insert.relation);
+        const row = {
+            ...insert.new,
+            [ZERO_VERSION_COLUMN_NAME]: this.#version,
+        };
+        const key = Object.fromEntries(insert.relation.keyColumns.map(col => [col, insert.new[col]]));
+        const rawColumns = Object.keys(row);
+        const keyColumns = insert.relation.keyColumns.map(c => ident(c));
+        const columns = rawColumns.map(c => ident(c));
+        const upsert = rawColumns.map(c => `${ident(c)}=EXCLUDED.${ident(c)}`);
+        this.#db.run(`
+      INSERT INTO ${ident(table)} (${columns.join(',')})
+        VALUES (${new Array(columns.length).fill('?').join(',')})
+        ON CONFLICT (${keyColumns.join(',')})
+        DO UPDATE SET ${upsert.join(',')}
+      `, liteValues(row));
+        logSetOp(this.#db, this.#version, table, key);
+    }
+    processUpdate(update) {
+        const table = liteTableName(update.relation);
+        const row = {
+            ...update.new,
+            [ZERO_VERSION_COLUMN_NAME]: this.#version,
+        };
+        // update.key is set with the old values if the key has changed.
+        const oldKey = update.key;
+        const newKey = Object.fromEntries(update.relation.keyColumns.map(col => [col, update.new[col]]));
+        const currKey = oldKey ?? newKey;
+        const setExprs = Object.keys(row).map(col => `${ident(col)}=?`);
+        const conds = Object.keys(currKey).map(col => `${ident(col)}=?`);
+        this.#db.run(`
+      UPDATE ${ident(table)}
+        SET ${setExprs.join(',')}
+        WHERE ${conds.join(' AND ')}
+      `, [...liteValues(row), ...liteValues(currKey)]);
+        if (oldKey) {
+            logDeleteOp(this.#db, this.#version, table, oldKey);
+        }
+        logSetOp(this.#db, this.#version, table, newKey);
+    }
+    processDelete(del) {
+        // REPLICA IDENTITY DEFAULT means the `key` must be set.
+        // https://www.postgresql.org/docs/current/protocol-logicalrep-message-formats.html
+        assert(del.relation.replicaIdentity === 'default');
+        assert(del.key);
+        const rowKey = del.key;
+        const table = liteTableName(del.relation);
+        const conds = Object.keys(rowKey).map(col => `${ident(col)}=?`);
+        this.#db.run(`DELETE FROM ${ident(table)} WHERE ${conds.join(' AND ')}`, liteValues(rowKey));
+        logDeleteOp(this.#db, this.#version, table, rowKey);
+    }
+    processTruncate(truncate) {
+        for (const relation of truncate.relations) {
+            const table = liteTableName(relation);
+            // Update replica data.
+            this.#db.run(`DELETE FROM ${ident(table)}`);
+            // Update change log.
+            logTruncateOp(this.#db, this.#version, table);
+        }
+    }
+    processCommit(commit, watermark) {
+        const nextVersion = watermark;
+        if (nextVersion <= this.#version) {
+            this.#db.rollback();
+            throw new ReplayedTransactionError(watermark, commit);
+        }
+        updateReplicationWatermark(this.#db, nextVersion);
+        this.#db.commit();
+        const elapsedMs = Date.now() - this.#startMs;
+        return elapsedMs;
+    }
+    abort(lc) {
+        lc.info?.(`aborting transaction ${this.#version}`);
+        this.#db.rollback();
+    }
+}
+//# sourceMappingURL=incremental-sync.js.map

package/out/zero-cache/src/services/replicator/incremental-sync.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"incremental-sync.js","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,KAAK,EAAC,MAAM,WAAW,CAAC;AAChC,OAAO,EAAC,yBAAyB,EAAC,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAC,UAAU,EAAC,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAC,MAAM,EAAE,WAAW,EAAC,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAC,eAAe,EAAC,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAC,UAAU,EAAC,MAAM,8BAA8B,CAAC;AACxD,OAAO,EAAC,QAAQ,EAAC,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAC,SAAS,EAAC,MAAM,4BAA4B,CAAC;AAErD,OAAO,EAAC,aAAa,EAAC,MAAM,sBAAsB,CAAC;AAcnD,OAAO,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAC,QAAQ,EAAC,MAAM,eAAe,CAAC;AAEvC,OAAO,EAAC,WAAW,EAAE,QAAQ,EAAE,aAAa,EAAC,MAAM,wBAAwB,CAAC;AAC5E,OAAO,EACL,wBAAwB,EACxB,sBAAsB,EACtB,oBAAoB,EACpB,0BAA0B,GAC3B,MAAM,+BAA+B,CAAC;AAEvC;;;;;GAKG;AACH,MAAM,OAAO,iBAAiB;IACnB,GAAG,CAAS;IACZ,eAAe,CAAiB;IAChC,QAAQ,CAAkB;IAC1B,SAAS,CAAW;IACpB,aAAa,CAAe;IAE5B,MAAM,GAAG,IAAI,YAAY,CAAC,mBAAmB,CAAC,CAAC;IACxD,QAAQ,CAAwC;IAEhD,YACE,EAAU,EACV,cAA8B,EAC9B,OAAiB,EACjB,YAA0B;QAE1B,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,QAAQ,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,EAAE,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAc;QACtB,EAAE,CAAC,IAAI,EAAE,CAAC,4BAA4B,CAAC,CAAC;QACxC,MAAM,EAAC,SAAS,EAAE,gBAAgB,EAAC,GAAG,oBAAoB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE1E,uEAAuE;QACvE,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,CAAC;QAEnC,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC;YAC/B,MAAM,EAAC,cAAc,EAAE,SAAS,EAAC,GAAG,oBAAoB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxE,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC;gBAChD,EAAE,EAAE,IAAI,CAAC,GAAG;gBACZ,SAAS;gBACT,cAAc;gBACd,OAAO,EAAE,SAAS,KAAK,gBAAgB;aACxC,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,IAAI,gBAAgB,CACpC,IAAI,CAAC,QAAQ,EACb,CAAC,UAAkB,EAAE,EAAE,GAAE,CAAC,EAAE,uCAAuC;YACnE,CAAC,EAAc,EAAE,GAAY,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CACrD,CAAC;YAEF,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;YACxD,IAAI,CAAC;gBACH,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,UAAU,EAAE,CAAC;oBACvC,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC;wBAC3B,yCAAyC;wBACzC,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;wBAChC,MAAM;oBACR,CAAC;oBACD,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;oBAE3B,MAAM,SAAS,GAAG,SAAS,CAAC,cAAc,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;oBACxD,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;wBAClB,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAC,KAAK,EAAE,eAAe,EAAC,CAAC,CAAC;wBAC3D,MAAM,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;oBACtE,CAAC;gBACH,CAAC;gBACD,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACtB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,EAAE,CAAC,KAAK,EAAE,CAAC,oCAAoC,EAAE,CAAC,CAAC,CAAC;gBACpD,SAAS,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;YACzB,CAAC;oBAAS,CAAC;gBACT,UAAU,CAAC,MAAM,EAAE,CAAC;gBACpB,UAAU,EAAE,CAAC;YACf,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAChC,CAAC;QACD,EAAE,CAAC,IAAI,EAAE,CAAC,2BAA2B,CAAC,CAAC;IACzC,CAAC;IAED,SAAS;QACP,OAAO,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAc,EAAE,GAAa;QACtC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1B,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;QAC1B,MAAM,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC9B,CAAC;CACF;AAED,SAAS,WAAW,CAAC,GAAY;IAC/B,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;QACzB,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC;IAClB,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,wBAAyB,SAAQ,KAAK;IACjC,SAAS,CAAS;IAE3B,YAAY,SAAiB,EAAE,MAAqB;QAClD,KAAK,CAAC,GAAG,SAAS,gCAAgC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;CACF;AAED;;;;;;;;;;GAUG;AACH,wBAAwB;AACxB,MAAM,OAAO,gBAAgB;IAClB,GAAG,CAAkB;IACrB,YAAY,CAAiC;IAC7C,YAAY,CAAyC;IAE9D,UAAU,GAAgC,IAAI,CAAC;IAC/C,eAAe,GAAG,CAAC,CAAC;IAEpB,QAAQ,CAAoB;IAE5B,YACE,EAAmB,EACnB,WAA2C,EAC3C,WAAmD;QAEnD,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QAChC,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,EAAc,EAAE,GAAY;QAChC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,qCAAqC;YAEjE,IAAI,CAAC,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;YAEjC,IAAI,GAAG,YAAY,UAAU,EAAE,CAAC;gBAC9B,0BAA0B;gBAC1B,EAAE,CAAC,IAAI,EAAE,CAAC,2BAA2B,CAAC,CAAC;YACzC,CAAC;iBAAM,CAAC;gBACN,2CAA2C;gBAC3C,EAAE,CAAC,KAAK,EAAE,CAAC,4BAA4B,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACxD,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,EAAc,EAAE,GAAa;QACjC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,+CAA+C;IAC/C,cAAc,CAAC,EAAc,EAAE,UAA4B;QACzD,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,UAAU,CAAC;QACnC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,EAAE,CAAC,KAAK,EAAE,CAAC,YAAY,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YACtC,OAAO,CAAC,CAAC;QACX,CAAC;QACD,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;YAC1E,OAAO,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QACtD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,YAAY,wBAAwB,EAAE,CAAC;gBAC1C,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;gBACb,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACjC,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,+CAA+C;IAC/C,eAAe,CACb,EAAc,EACd,GAAW,EACX,SAA6B;QAE7B,IAAI,GAAG,CAAC,GAAG,KAAK,OAAO,EAAE,CAAC;YACxB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,4BAA4B,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChE,CAAC;YACD,IAAI,CAAC,UAAU,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrD,OAAO,CAAC,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC;QACpC,CAAC;QAED,4DAA4D;QAC5D,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC;QAC3B,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,MAAM,IAAI,KAAK,CACb,4CAA4C,SAAS,CAAC,GAAG,CAAC,EAAE,CAC7D,CAAC;QACJ,CAAC;QAED,IAAI,GAAG,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;YACzB,uEAAuE;YACvE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YAEvB,MAAM,CAAC,SAAS,CAAC,CAAC;YAClB,MAAM,SAAS,GAAG,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YACnD,EAAE,CAAC,KAAK,EAAE,CAAC,iBAAiB,SAAS,MAAM,CAAC,CAAC;YAE7C,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;YAC7B,OAAO,IAAI,CAAC,eAAe,CAAC;QAC9B,CAAC;QAED,IAAI,CAAC,eAAe,EAAE,CAAC;QAEvB,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC;YAChB,KAAK,QAAQ;gBACX,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;gBACtB,MAAM;YACR,KAAK,QAAQ;gBACX,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;gBACtB,MAAM;YACR,KAAK,QAAQ;gBACX,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;gBACtB,MAAM;YACR,KAAK,UAAU;gBACb,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;gBACxB,MAAM;YAER;gBACE,WAAW,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;QAED,OAAO,CAAC,CAAC;IACX,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,oBAAoB;IACf,QAAQ,CAAS;IACjB,GAAG,CAAkB;IACrB,QAAQ,CAAc;IAE/B,YAAY,EAAmB;QAC7B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE3B,kFAAkF;QAClF,6EAA6E;QAC7E,8EAA8E;QAC9E,mBAAmB;QACnB,EAAE;QACF,iEAAiE;QACjE,4CAA4C;QAC5C,EAAE,CAAC,eAAe,EAAE,CAAC;QACrB,MAAM,EAAC,gBAAgB,EAAC,GAAG,sBAAsB,CAAC,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,CAAC,QAAQ,GAAG,gBAAgB,CAAC;IACnC,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,aAAa,CAAC,MAAqB;QACjC,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,GAAG,GAAG;YACV,GAAG,MAAM,CAAC,GAAG;YACb,CAAC,wBAAwB,CAAC,EAAE,IAAI,CAAC,QAAQ;SAC1C,CAAC;QACF,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAC5B,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAC9D,CAAC;QACF,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,GAAG,CAAC,GAAG,CACV;oBACc,KAAK,CAAC,KAAK,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;kBACpC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;uBACxC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;wBACnB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;OACjC,EACD,UAAU,CAAC,GAAG,CAAC,CAChB,CAAC;QAEF,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IAChD,CAAC;IAED,aAAa,CAAC,MAAqB;QACjC,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,GAAG,GAAG;YACV,GAAG,MAAM,CAAC,GAAG;YACb,CAAC,wBAAwB,CAAC,EAAE,IAAI,CAAC,QAAQ;SAC1C,CAAC;QACF,gEAAgE;QAChE,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAC/B,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAC9D,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,IAAI,MAAM,CAAC;QACjC,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAChE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEjE,IAAI,CAAC,GAAG,CAAC,GAAG,CACV;eACS,KAAK,CAAC,KAAK,CAAC;cACb,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;gBAChB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC;OAC5B,EACD,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAC7C,CAAC;QAEF,IAAI,MAAM,EAAE,CAAC;YACX,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QACtD,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACnD,CAAC;IAED,aAAa,CAAC,GAAkB;QAC9B,wDAAwD;QACxD,mFAAmF;QACnF,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC;QACnD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChB,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;QACvB,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE1C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAChE,IAAI,CAAC,GAAG,CAAC,GAAG,CACV,eAAe,KAAK,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAC1D,UAAU,CAAC,MAAM,CAAC,CACnB,CAAC;QAEF,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACtD,CAAC;IAED,eAAe,CAAC,QAAyB;QACvC,KAAK,MAAM,QAAQ,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;YACtC,uBAAuB;YACvB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,eAAe,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAE5C,qBAAqB;YACrB,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,aAAa,CAAC,MAAqB,EAAE,SAAiB;QACpD,MAAM,WAAW,GAAG,SAAS,CAAC;QAC9B,IAAI,WAAW,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YACpB,MAAM,IAAI,wBAAwB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACxD,CAAC;QACD,0BAA0B,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAClD,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;QAElB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC7C,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,EAAc;QAClB,EAAE,CAAC,IAAI,EAAE,CAAC,wBAAwB,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnD,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;IACtB,CAAC;CACF"}

package/out/zero-cache/src/services/replicator/notifier.d.ts

@@ -0,0 +1,30 @@
+import { type Result, Subscription } from '../../types/subscription.js';
+import type { ReplicaState, ReplicaStateNotifier } from './replicator.js';
+/**
+ * Handles the semantics of {@link ReplicatorVersionNotifier.subscribe()}
+ * notifications, namely:
+ *
+ * * New subscribers are notified immediately with the latest received
+ *   ReplicaState.
+ *
+ * * Non-latest notifications are discarded if the subscriber is too
+ *   busy to consume them.
+ *
+ * By encapsulating the state for the first behavior (essentially, whether
+ * the first notification has been sent by the Replicator), Notifier objects
+ * can be chained to simplify fanout from Replicator to View Syncers.
+ *
+ * In particular, each Syncer Thread can manage a single Subscription to
+ * the Replicator across a MessageChannel, which it uses for its own Notifier
+ * instance to manage subscriptions from View Syncers within its thread. This
+ * way a Replicator only deals with sending notifications to a bounded set
+ * of MessageChannel-based subscribers (Syncer Threads), while the dynamic
+ * subscribe and unsubscribe traffic from View Syncers remains within each
+ * Syncer Thread.
+ */
+export declare class Notifier implements ReplicaStateNotifier {
+    #private;
+    subscribe(): Subscription<ReplicaState>;
+    notifySubscribers(state?: ReplicaState): Promise<Result>[];
+}
+//# sourceMappingURL=notifier.d.ts.map

package/out/zero-cache/src/services/replicator/notifier.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"notifier.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/notifier.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,MAAM,EACX,YAAY,EACb,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAC,YAAY,EAAE,oBAAoB,EAAC,MAAM,iBAAiB,CAAC;AAExE;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,QAAS,YAAW,oBAAoB;;IAanD,SAAS,IAAI,YAAY,CAAC,YAAY,CAAC;IAWvC,iBAAiB,CACf,KAAK,GAAE,YAAuC,GAC7C,OAAO,CAAC,MAAM,CAAC,EAAE;CAOrB"}