@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.41

package/out/zero-cache/src/services/change-streamer/storer.d.ts

@@ -9,13 +9,37 @@ import type { UpstreamStatusMessage } from '../change-source/protocol/current/st
 import { type CookieSet } from '../replicator/change-log-cookies.ts';
 import type { ReplicatorMode } from '../replicator/replicator.ts';
 import type { Service } from '../service.ts';
-import { type ChangeLogEntry } from './change-log-codec.ts';
+import { type ChangeLogEntry, type SerializedChangeStreamData } from './change-log-codec.ts';
+import { type ReplicaSubscriptionState, type ChangeLogReinitialization } from './schema/tables.ts';
 import type { Subscriber } from './subscriber.ts';
 /**
  * Factory for creating dynamic connection pools used by the Storer to
  * isolate concurrent transactions into separate pg clients.
  */
 export type PostgresDBProvider = (applicationName: string, maxConns: number) => PostgresDB;
+export type OwnershipTakeover = {
+    /**
+     * The lock that prevents the previous owner from purging the change log
+     * (from which this change-streamer's replica was restored), released once
+     * ownership is assumed.
+     */
+    purgeLock: PurgeLock | null;
+    /**
+     * Requests a restart of the change stream (at a transaction boundary),
+     * which is used to start pending backfills once ownership is assumed.
+     */
+    requestRestart: (reason: string) => void;
+    /**
+     * Whether to throw an AutoResetSignal if the change db requires a reset.
+     */
+    autoReset: boolean;
+    /**
+     * Whether the change log was behind the replication slot, in which case it
+     * is re-initialized from the replica (see `reinitializeFromReplica()`), and
+     * can thus be reset for a replica at any watermark.
+     */
+    pgChangeLogBehindSlot: boolean;
+};
 export type TuningOptions = {
     backPressureLimitHeapProportion: number;
     statementTimeoutMs: number;
@@ -55,8 +79,66 @@ export type TuningOptions = {
 export declare class Storer implements Service {
     #private;
     readonly id = "storer";
-    constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, dbProvider: PostgresDBProvider, replicaVersion: string, onCommitted: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs, }: TuningOptions);
-    assumeOwnership(purgeLock?: PurgeLock | null): Promise<void>;
+    constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, dbProvider: PostgresDBProvider, subscriptionState: ReplicaSubscriptionState, onCommitted: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs, }: TuningOptions);
+    /**
+     * Re-initializes a change log that is no longer maintained from the
+     * replica's state, taking ownership of it in the process (which is then
+     * assumed again at its new head, see {@link assumeOwnershipAtHead}).
+     * See {@link reinitializeChangeLog}.
+     */
+    reinitializeFromReplica(params: Omit<ChangeLogReinitialization, 'owner' | 'ownerAddress'>): Promise<'reinitialized' | 'maintained'>;
+    /**
+     * Prepares to assume ownership of the change log at its head, while it may
+     * still be owned and maintained by another change-streamer. This allows the
+     * previous owner to continue to serve subscribers while this change-source's
+     * upstream decodes the WAL to catch up to the head of the change log, which
+     * may be many minutes worth of transactions for large DBs.
+     *
+     * Once the change stream is started from the head of the change log:
+     *
+     * * Transactions that the change log already contains (i.e. stored by the
+     *   previous owner) are not stored again, but are otherwise processed
+     *   normally (e.g. for upstream acks, since they have been stored).
+     * * Ownership is assumed in the transaction that stores the first
+     *   transaction past the head of the change log. The previous owner's
+     *   (subsequent) transactions fail its ownership check, which makes this
+     *   transaction the one immediately after the previous owner's last one.
+     * * Ownership is also assumed upon a {@link status} at or past the head of
+     *   the change log (outside of a transaction), which covers an idle
+     *   upstream: all transactions up to the status position have been
+     *   received, and any past the head would have been stored.
+     *
+     * Subscribers are caught up from the change log as usual, as the stream
+     * (which is forwarded to them) starts from its head.
+     *
+     * Backfill transactions are synthesized by each change-streamer's change
+     * source, and thus never correspond to those stored by the previous owner.
+     * Until ownership is assumed, they are not stored, and no backfills are
+     * requested from the change log (see
+     * {@link getStartStreamInitializationParameters}). Once ownership is
+     * assumed, a restart of the change stream is requested if the change log has
+     * pending backfills or if a backfill transaction was skipped.
+     */
+    assumeOwnershipAtHead(takeover: OwnershipTakeover): Promise<void>;
+    /**
+     * Whether ownership of the change log is pending, i.e. after
+     * {@link assumeOwnershipAtHead} and before ownership is assumed.
+     */
+    get ownershipPending(): boolean;
+    /**
+     * Relays a status message received outside of a transaction, which is
+     * used to assume ownership of the change log on an idle upstream. A no-op
+     * unless ownership is pending.
+     */
+    status(watermark: string): void;
+    /**
+     * Assumes ownership of the change log immediately, regardless of its head.
+     * This is used for in tests.
+     *
+     * A production change-streamer instead takes over the change log at its head
+     * (see {@link assumeOwnershipAtHead}).
+     */
+    assumeOwnership(): Promise<void>;
     /**
      * The whole of what a stream connection starts from: where to resume, what to
      * ask the change source to backfill, and the raw cookie set those requests
@@ -106,7 +188,7 @@ export declare class Storer implements Service {
     /**
      * @returns The JSON stringified stream message to be sent downstream.
      */
-    store(watermark: string, data: ChangeStreamData): string;
+    store(watermark: string, data: ChangeStreamData, serialized?: SerializedChangeStreamData): string;
     abort(): void;
     catchup(subscriber: Subscriber, mode: ReplicatorMode): void;
     readyForMore(): Promise<void> | undefined;
@@ -131,13 +213,32 @@ export declare class PurgeLock {
     #private;
     readonly replicaVersion: string;
     readonly minWatermark: string;
-    constructor(lc: LogContext, tx: TransactionPool, replicaVersion: string, watermark: string);
+    /**
+     * The head of the change log (i.e. its `lastWatermark`) when the lock was
+     * acquired. The change log is resumed from its head, which only advances,
+     * so a stream cannot (correctly) be started from before it.
+     */
+    readonly headWatermark: string;
+    constructor(lc: LogContext, tx: TransactionPool, replicaVersion: string, watermark: string, headWatermark: string);
     release(): Promise<void>;
 }
 export declare class PurgeLocker {
     #private;
     constructor(lc: LogContext, shard: ShardID, db: PostgresDB);
+    /**
+     * Locks the oldest row of the change log, preventing it from being purged.
+     *
+     * @param slotWatermark The position of the replication slot from which the
+     *     change log is to be resumed (as a watermark). If specified and the
+     *     head of the change log is behind it, the change log may be missing
+     *     transactions that the slot will not stream, in which case nothing is
+     *     locked and `'behind-slot'` is returned. Conversely, a head at or past
+     *     the slot's position means that the stream (which resumes from the
+     *     head) will not skip anything.
+     * @returns `null` if the change log is empty.
+     */
     acquire(): Promise<PurgeLock | null>;
+    acquire(slotWatermark?: string | undefined): Promise<PurgeLock | null | 'behind-slot'>;
 }
 type TaskKey = JSONObject;
 type TaskDoneFn = () => void;

package/out/zero-cache/src/services/change-streamer/storer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,uCAAuC,CAAC;AAM/C,OAAO,EAAiB,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EAMrB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,6CAA6C,CAAC;AACvF,OAAO,EAIL,KAAK,SAAS,EAEf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,EAIL,KAAK,cAAc,EACpB,MAAM,uBAAuB,CAAC;AAQ/B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAC/B,eAAe,EAAE,MAAM,EACvB,QAAQ,EAAE,MAAM,KACb,UAAU,CAAC;AAsChB,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;IA4BvB,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,UAAU,EAAE,kBAAkB,EAC9B,cAAc,EAAE,MAAM,EACtB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACxD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa,EAkCjB;IAgCK,eAAe,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI,iBA4BjD;IAED;;;;;;;;;;;;;;;;OAgBG;IACG,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;QACpC,OAAO,EAAE,SAAS,CAAC;KACpB,CAAC,CAsDD;IAEK,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAOxD;IAED;;;;OAIG;IACG,gBAAgB,IAAI,OAAO,CAAC;QAChC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CASD;IAED;;;OAGG;IACG,oBAAoB,CACxB,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,EAAE,CAAC,CASnB;IAED;;;;;;OAMG;IACH,gBAAgB,CACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,SAAS,SAAO,GACf,aAAa,CAAC,cAAc,EAAE,CAAC,CAYjC;IAMD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAuCrD;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,UAgB9C;IAED,KAAK,SAEJ;IAED,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc,QAEnD;IAID,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAwBxC;IAmDD;;;OAGG;IACG,GAAG,kBA6BR;IAmfD;;;OAGG;IACG,YAAY,kBAMjB;IAED;;;;OAIG;IACG,IAAI,kBAcT;CACF;AAED,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAE9B,YACE,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,EAMlB;IAIK,OAAO,kBAUZ;CACF;AAED,qBAAa,WAAW;;IAKtB,YAAY,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU,EAIzD;IAOK,OAAO,8BA2BZ;CACF;AAKD,KAAK,OAAO,GAAG,UAAU,CAAC;AAE1B,KAAK,UAAU,GAAG,MAAM,IAAI,CAAC;AAE7B;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,eAAe;;IAQ1B,YACE,EAAE,EAAE,UAAU,EACd,kBAAkB,EAAE,MAAM,EAC1B,SAAS,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAKhC;IAED,KAAK,SAMJ;IAGD,iBAAiB,CAAC,GAAG,EAAE,IAAI,QAgB1B;IAED;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,OAAa,GAAG,UAAU,CAKrD;IAED,IAAI,SAGH;CACF"}
+{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,uCAAuC,CAAC;AAO/C,OAAO,EAAiB,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EAMrB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,6CAA6C,CAAC;AACvF,OAAO,EAIL,KAAK,SAAS,EAEf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,0BAA0B,EAChC,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAKL,KAAK,wBAAwB,EAE7B,KAAK,yBAAyB,EAE/B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAC/B,eAAe,EAAE,MAAM,EACvB,QAAQ,EAAE,MAAM,KACb,UAAU,CAAC;AAsDhB,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;;;OAIG;IACH,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;IAC5B;;;OAGG;IACH,cAAc,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IACzC;;OAEG;IACH,SAAS,EAAE,OAAO,CAAC;IACnB;;;;OAIG;IACH,qBAAqB,EAAE,OAAO,CAAC;CAChC,CAAC;AAQF,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;IA2CvB,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,UAAU,EAAE,kBAAkB,EAC9B,iBAAiB,EAAE,wBAAwB,EAC3C,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACxD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa,EAmCjB;IAyCD;;;;;OAKG;IACH,uBAAuB,CACrB,MAAM,EAAE,IAAI,CAAC,yBAAyB,EAAE,OAAO,GAAG,cAAc,CAAC,GAChE,OAAO,CAAC,eAAe,GAAG,YAAY,CAAC,CAUzC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACG,qBAAqB,CAAC,QAAQ,EAAE,iBAAiB,iBAqBtD;IAED;;;OAGG;IACH,IAAI,gBAAgB,IAAI,OAAO,CAE9B;IAED;;;;OAIG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,QAIvB;IA6DD;;;;;;OAMG;IACG,eAAe,kBAcpB;IAED;;;;;;;;;;;;;;;;OAgBG;IACG,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;QACpC,OAAO,EAAE,SAAS,CAAC;KACpB,CAAC,CA4DD;IAEK,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAOxD;IAED;;;;OAIG;IACG,gBAAgB,IAAI,OAAO,CAAC;QAChC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CASD;IAED;;;OAGG;IACG,oBAAoB,CACxB,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,EAAE,CAAC,CASnB;IAED;;;;;;OAMG;IACH,gBAAgB,CACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,SAAS,SAAO,GACf,aAAa,CAAC,cAAc,EAAE,CAAC,CAYjC;IAMD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAuCrD;IAED;;OAEG;IACH,KAAK,CACH,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,gBAAgB,EACtB,UAAU,GAAE,0BAEX,UAkBF;IAED,KAAK,SAEJ;IAED,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc,QAEnD;IAID,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAwBxC;IAmDD;;;OAGG;IACG,GAAG,kBA6BR;IA6kBD;;;OAGG;IACG,YAAY,kBAMjB;IAED;;;;OAIG;IACG,IAAI,kBAcT;CACF;AAED,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;OAIG;IACH,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAE/B,YACE,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EAOtB;IAIK,OAAO,kBAUZ;CACF;AAED,qBAAa,WAAW;;IAKtB,YAAY,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU,EAIzD;IAOD;;;;;;;;;;;OAWG;IACH,OAAO,IAAI,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;IACrC,OAAO,CACL,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,GACjC,OAAO,CAAC,SAAS,GAAG,IAAI,GAAG,aAAa,CAAC,CAAC;CA6D9C;AAKD,KAAK,OAAO,GAAG,UAAU,CAAC;AAE1B,KAAK,UAAU,GAAG,MAAM,IAAI,CAAC;AAE7B;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,eAAe;;IAQ1B,YACE,EAAE,EAAE,UAAU,EACd,kBAAkB,EAAE,MAAM,EAC1B,SAAS,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAKhC;IAED,KAAK,SAMJ;IAGD,iBAAiB,CAAC,GAAG,EAAE,IAAI,QAgB1B;IAED;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,OAAa,GAAG,UAAU,CAKrD;IAED,IAAI,SAGH;CACF"}