@rocicorp/zero 1.12.0-canary.6 → 1.12.0-canary.7

package/out/zero-cache/src/services/change-streamer/storer.d.ts

@@ -1,276 +0,0 @@
-import type { LogContext } from '@rocicorp/logger';
-import { type JSONObject } from '../../../../shared/src/bigint-json.ts';
-import { TransactionPool } from '../../db/transaction-pool.ts';
-import { type PostgresDB } from '../../types/pg.ts';
-import { type ShardID } from '../../types/shards.ts';
-import { type BackfillRequest } from '../change-source/protocol/current.ts';
-import { type ChangeStreamData, type Commit } from '../change-source/protocol/current/downstream.ts';
-import type { UpstreamStatusMessage } from '../change-source/protocol/current/status.ts';
-import { type CookieSet } from '../replicator/change-log-cookies.ts';
-import type { ReplicatorMode } from '../replicator/replicator.ts';
-import type { Service } from '../service.ts';
-import { type ChangeLogEntry, type SerializedChangeStreamData } from './change-log-codec.ts';
-import { type ReplicaSubscriptionState, type ChangeLogReinitialization } from './schema/tables.ts';
-import type { Subscriber } from './subscriber.ts';
-/**
- * Factory for creating dynamic connection pools used by the Storer to
- * isolate concurrent transactions into separate pg clients.
- */
-export type PostgresDBProvider = (applicationName: string, maxConns: number) => PostgresDB;
-export type OwnershipTakeover = {
-    /**
-     * The lock that prevents the previous owner from purging the change log
-     * (from which this change-streamer's replica was restored), released once
-     * ownership is assumed.
-     */
-    purgeLock: PurgeLock | null;
-    /**
-     * Requests a restart of the change stream (at a transaction boundary),
-     * which is used to start pending backfills once ownership is assumed.
-     */
-    requestRestart: (reason: string) => void;
-    /**
-     * Whether to throw an AutoResetSignal if the change db requires a reset.
-     */
-    autoReset: boolean;
-    /**
-     * Whether the change log was behind the replication slot, in which case it
-     * is re-initialized from the replica (see `reinitializeFromReplica()`), and
-     * can thus be reset for a replica at any watermark.
-     */
-    pgChangeLogBehindSlot: boolean;
-};
-export type TuningOptions = {
-    backPressureLimitHeapProportion: number;
-    statementTimeoutMs: number;
-    changeLogBatchSize: number;
-    drainTimeoutMs?: number | undefined;
-};
-/**
- * Handles the storage of changes and the catchup of subscribers
- * that are behind.
- *
- * In the context of catchup and cleanup, it is the responsibility of the
- * Storer to decide whether a client can be caught up, or whether the
- * changes needed to catch a client up have been purged.
- *
- * **Maintained invariant**: The Change DB is only empty for a
- * completely new replica (i.e. initial-sync with no changes from the
- * replication stream).
- * * In this case, all new subscribers are expected start from the
- *   `replicaVersion`, which is the version at which initial sync
- *   was performed, and any attempts to catchup from a different
- *   point fail.
- *
- * Conversely, if non-initial changes have flowed through the system
- * (i.e. via the replication stream), the ChangeDB must *not* be empty,
- * and the earliest change in the `changeLog` represents the earliest
- * "commit" from (after) which a subscriber can be caught up.
- * * Any attempts to catchup from an earlier point must fail with
- *   a `WatermarkTooOld` error.
- * * Failure to do so could result in streaming changes to the
- *   subscriber such that there is a gap in its replication history.
- *
- * Note: Subscribers (i.e. `incremental-syncer`) consider an "error" signal
- * an unrecoverable error and shut down in response. This allows the
- * production system to replace it with a new task and fresh copy of the
- * replica backup.
- */
-export declare class Storer implements Service {
-    #private;
-    readonly id = "storer";
-    constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, dbProvider: PostgresDBProvider, subscriptionState: ReplicaSubscriptionState, onCommitted: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs, }: TuningOptions);
-    /**
-     * Re-initializes a change log that is no longer maintained from the
-     * replica's state, taking ownership of it in the process (which is then
-     * assumed again at its new head, see {@link assumeOwnershipAtHead}).
-     * See {@link reinitializeChangeLog}.
-     */
-    reinitializeFromReplica(params: Omit<ChangeLogReinitialization, 'owner' | 'ownerAddress'>): Promise<'reinitialized' | 'maintained'>;
-    /**
-     * Prepares to assume ownership of the change log at its head, while it may
-     * still be owned and maintained by another change-streamer. This allows the
-     * previous owner to continue to serve subscribers while this change-source's
-     * upstream decodes the WAL to catch up to the head of the change log, which
-     * may be many minutes worth of transactions for large DBs.
-     *
-     * Once the change stream is started from the head of the change log:
-     *
-     * * Transactions that the change log already contains (i.e. stored by the
-     *   previous owner) are not stored again, but are otherwise processed
-     *   normally (e.g. for upstream acks, since they have been stored).
-     * * Ownership is assumed in the transaction that stores the first
-     *   transaction past the head of the change log. The previous owner's
-     *   (subsequent) transactions fail its ownership check, which makes this
-     *   transaction the one immediately after the previous owner's last one.
-     * * Ownership is also assumed upon a {@link status} at or past the head of
-     *   the change log (outside of a transaction), which covers an idle
-     *   upstream: all transactions up to the status position have been
-     *   received, and any past the head would have been stored.
-     *
-     * Subscribers are caught up from the change log as usual, as the stream
-     * (which is forwarded to them) starts from its head.
-     *
-     * Backfill transactions are synthesized by each change-streamer's change
-     * source, and thus never correspond to those stored by the previous owner.
-     * Until ownership is assumed, they are not stored, and no backfills are
-     * requested from the change log (see
-     * {@link getStartStreamInitializationParameters}). Once ownership is
-     * assumed, a restart of the change stream is requested if the change log has
-     * pending backfills or if a backfill transaction was skipped.
-     */
-    assumeOwnershipAtHead(takeover: OwnershipTakeover): Promise<void>;
-    /**
-     * Whether ownership of the change log is pending, i.e. after
-     * {@link assumeOwnershipAtHead} and before ownership is assumed.
-     */
-    get ownershipPending(): boolean;
-    /**
-     * Relays a status message received outside of a transaction, which is
-     * used to assume ownership of the change log on an idle upstream. A no-op
-     * unless ownership is pending.
-     */
-    status(watermark: string): void;
-    /**
-     * Assumes ownership of the change log immediately, regardless of its head.
-     * This is used for in tests.
-     *
-     * A production change-streamer instead takes over the change log at its head
-     * (see {@link assumeOwnershipAtHead}).
-     */
-    assumeOwnership(): Promise<void>;
-    /**
-     * The whole of what a stream connection starts from: where to resume, what to
-     * ask the change source to backfill, and the raw cookie set those requests
-     * were derived from.
-     *
-     * All four statements run in one REPEATABLE READ transaction, so they see one
-     * snapshot. That is invariant 15 — the resume watermark and the cookies must
-     * come from the same store, read at the same position — and it is why the
-     * cookies are read here rather than by a second call: a cookie set paired
-     * with a watermark it was not folded to loses every backfill that completed
-     * in the interval, and those rows are not replayable from the slot.
-     *
-     * `cookies` is not `backfillRequests` in another shape. The request list is
-     * driven off `backfilling`, so it drops the metadata of every table with no
-     * in-flight backfill — correct for starting a stream, lossy as the cookie
-     * snapshot that the SQLite change log continues folding onto.
-     */
-    getStartStreamInitializationParameters(): Promise<{
-        lastWatermark: string;
-        backfillRequests: BackfillRequest[];
-        cookies: CookieSet;
-    }>;
-    getMinWatermarkForCatchup(): Promise<string | null>;
-    /**
-     * Returns the retained Postgres catchup bounds.
-     * `minWatermark` is the oldest transaction, or `null` for a new replica.
-     * `lastWatermark` is the durable head.
-     */
-    getCatchupBounds(): Promise<{
-        minWatermark: string | null;
-        lastWatermark: string;
-    }>;
-    /**
-     * Lists at most `limit` committed transaction watermarks in
-     * `(afterWatermark, throughWatermark]`, in ascending order.
-     */
-    listCommitWatermarks(afterWatermark: string, throughWatermark: string, limit: number): Promise<string[]>;
-    /**
-     * Reads `(afterWatermark, throughWatermark]` in stream order and bounded batches.
-     * This query matches the subscriber catchup query.
-     *
-     * Both queries fail when an escaped NUL reaches `change->'tag'`.
-     * Postgres cannot convert that value to text.
-     */
-    readCatchupRange(afterWatermark: string, throughWatermark: string, batchRows?: number): AsyncIterable<ChangeLogEntry[]>;
-    purgeRecordsBefore(watermark: string): Promise<number>;
-    /**
-     * @returns The JSON stringified stream message to be sent downstream.
-     */
-    store(watermark: string, data: ChangeStreamData, serialized?: SerializedChangeStreamData): string;
-    abort(): void;
-    catchup(subscriber: Subscriber, mode: ReplicatorMode): void;
-    readyForMore(): Promise<void> | undefined;
-    /**
-     * Runs the storer loop until {@link stop()} is called, or an error is thrown.
-     * Once {@link run()} completes, it can be called again.
-     */
-    run(): Promise<void>;
-    /**
-     * Waits until all currently queued entries have been processed.
-     * This is only used in tests.
-     */
-    allProcessed(): Promise<void>;
-    /**
-     * Stops the storer and waits up to a drain timeout for entries to drain,
-     * throwing an exception if it doesn't drain in time, in order to abort
-     * the server when PG (or the connection to it) appears to be wedged.
-     */
-    stop(): Promise<void>;
-}
-export declare class PurgeLock {
-    #private;
-    readonly replicaVersion: string;
-    readonly minWatermark: string;
-    /**
-     * The head of the change log (i.e. its `lastWatermark`) when the lock was
-     * acquired. The change log is resumed from its head, which only advances,
-     * so a stream cannot (correctly) be started from before it.
-     */
-    readonly headWatermark: string;
-    constructor(lc: LogContext, tx: TransactionPool, replicaVersion: string, watermark: string, headWatermark: string);
-    release(): Promise<void>;
-}
-export declare class PurgeLocker {
-    #private;
-    constructor(lc: LogContext, shard: ShardID, db: PostgresDB);
-    /**
-     * Locks the oldest row of the change log, preventing it from being purged.
-     *
-     * @param slotWatermark The position of the replication slot from which the
-     *     change log is to be resumed (as a watermark). If specified and the
-     *     head of the change log is behind it, the change log may be missing
-     *     transactions that the slot will not stream, in which case nothing is
-     *     locked and `'behind-slot'` is returned. Conversely, a head at or past
-     *     the slot's position means that the stream (which resumes from the
-     *     head) will not skip anything.
-     * @returns `null` if the change log is empty.
-     */
-    acquire(): Promise<PurgeLock | null>;
-    acquire(slotWatermark?: string | undefined): Promise<PurgeLock | null | 'behind-slot'>;
-}
-type TaskKey = JSONObject;
-type TaskDoneFn = () => void;
-/**
- * Periodic monitor for detecting the pathological condition in which a db
- * query hangs forever because of a connection problem, such as a half-closed
- * socket.
- *
- * Use this for db calls that recur or accumulate progress within a single
- * logical operation on the main storer loop (e.g. the queue-processing loop,
- * or a multi-batch catchup read), where "no progress for N ms" is the
- * meaningful signal. For a one-off db call -- including ones made before
- * {@link Storer.run} starts this monitor's polling -- use a plain timeout
- * (see `Storer.#withTimeout`) instead.
- *
- * The failure handler should stop the server. The ProgressMonitor will
- * assume this and stop itself after calling it.
- *
- * Internal to the Storer, but exported for testing.
- */
-export declare class ProgressMonitor {
-    #private;
-    constructor(lc: LogContext, failureThresholdMs: number, onFailure: (err: Error) => void);
-    start(): void;
-    checkTaskProgress(now: Date): void;
-    /**
-     * Starts a task to be tracked.
-     *
-     * @return a callback to invoke when the task is done.
-     */
-    trackTask(task: TaskKey, now?: Date): TaskDoneFn;
-    stop(): void;
-}
-export {};
-//# sourceMappingURL=storer.d.ts.map

package/out/zero-cache/src/services/change-streamer/storer.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,uCAAuC,CAAC;AAO/C,OAAO,EAAiB,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EAMrB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,6CAA6C,CAAC;AACvF,OAAO,EAIL,KAAK,SAAS,EAEf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,0BAA0B,EAChC,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAKL,KAAK,wBAAwB,EAE7B,KAAK,yBAAyB,EAE/B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAC/B,eAAe,EAAE,MAAM,EACvB,QAAQ,EAAE,MAAM,KACb,UAAU,CAAC;AAsDhB,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;;;OAIG;IACH,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;IAC5B;;;OAGG;IACH,cAAc,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IACzC;;OAEG;IACH,SAAS,EAAE,OAAO,CAAC;IACnB;;;;OAIG;IACH,qBAAqB,EAAE,OAAO,CAAC;CAChC,CAAC;AAQF,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;IA2CvB,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,UAAU,EAAE,kBAAkB,EAC9B,iBAAiB,EAAE,wBAAwB,EAC3C,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACxD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa,EAmCjB;IAyCD;;;;;OAKG;IACH,uBAAuB,CACrB,MAAM,EAAE,IAAI,CAAC,yBAAyB,EAAE,OAAO,GAAG,cAAc,CAAC,GAChE,OAAO,CAAC,eAAe,GAAG,YAAY,CAAC,CAUzC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACG,qBAAqB,CAAC,QAAQ,EAAE,iBAAiB,iBAqBtD;IAED;;;OAGG;IACH,IAAI,gBAAgB,IAAI,OAAO,CAE9B;IAED;;;;OAIG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,QAIvB;IA6DD;;;;;;OAMG;IACG,eAAe,kBAcpB;IAED;;;;;;;;;;;;;;;;OAgBG;IACG,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;QACpC,OAAO,EAAE,SAAS,CAAC;KACpB,CAAC,CA4DD;IAEK,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAOxD;IAED;;;;OAIG;IACG,gBAAgB,IAAI,OAAO,CAAC;QAChC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CASD;IAED;;;OAGG;IACG,oBAAoB,CACxB,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,EAAE,CAAC,CASnB;IAED;;;;;;OAMG;IACH,gBAAgB,CACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,SAAS,SAAO,GACf,aAAa,CAAC,cAAc,EAAE,CAAC,CAYjC;IAMD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAuCrD;IAED;;OAEG;IACH,KAAK,CACH,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,gBAAgB,EACtB,UAAU,GAAE,0BAEX,UAkBF;IAED,KAAK,SAEJ;IAED,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc,QAEnD;IAID,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAwBxC;IAmDD;;;OAGG;IACG,GAAG,kBA6BR;IAilBD;;;OAGG;IACG,YAAY,kBAMjB;IAED;;;;OAIG;IACG,IAAI,kBAcT;CACF;AAED,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;OAIG;IACH,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAE/B,YACE,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EAOtB;IAIK,OAAO,kBAUZ;CACF;AAED,qBAAa,WAAW;;IAKtB,YAAY,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU,EAIzD;IAOD;;;;;;;;;;;OAWG;IACH,OAAO,IAAI,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;IACrC,OAAO,CACL,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,GACjC,OAAO,CAAC,SAAS,GAAG,IAAI,GAAG,aAAa,CAAC,CAAC;CA6D9C;AAKD,KAAK,OAAO,GAAG,UAAU,CAAC;AAE1B,KAAK,UAAU,GAAG,MAAM,IAAI,CAAC;AAE7B;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,eAAe;;IAQ1B,YACE,EAAE,EAAE,UAAU,EACd,kBAAkB,EAAE,MAAM,EAC1B,SAAS,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAKhC;IAED,KAAK,SAMJ;IAGD,iBAAiB,CAAC,GAAG,EAAE,IAAI,QAgB1B;IAED;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,OAAa,GAAG,UAAU,CAKrD;IAED,IAAI,SAGH;CACF"}