@rocicorp/zero 1.10.0-canary.0 → 1.10.0-canary.10

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts

@@ -2,36 +2,102 @@ import type { LogContext } from '@rocicorp/logger';
 import type { PostgresDB } from '../../types/pg.ts';
 import type { ShardID } from '../../types/shards.ts';
 import type { ChangeSource } from '../change-source/change-source.ts';
-import { type ChangeStreamData } from '../change-source/protocol/current/downstream.ts';
+import type { LitestreamVersion } from '../litestream/metrics.ts';
 import { type ReplicationStatusPublisher } from '../replicator/replication-status.ts';
 import type { SubscriptionState } from '../replicator/schema/replication-state.ts';
-import { type ChangeStreamerService } from './change-streamer.ts';
+import { type ChangeStreamerService, type SubscriberContext } from './change-streamer.ts';
+import { type SQLiteChangeLogCleanupGuard } from './sqlite-change-log-catchup.ts';
+import { type SQLiteChangeLogCompareOptions } from './sqlite-change-log-comparator.ts';
+import { type SQLiteChangeLogPurgeSchedulerOptions } from './sqlite-change-log-purge-scheduler.ts';
+import { type SQLiteChangeLogCoverage } from './sqlite-change-log-read-router.ts';
+import { type SQLiteChangeLogWriterOptions } from './sqlite-change-log-writer.ts';
 import { type PurgeLock, type TuningOptions as StorerOptions } from './storer.ts';
+export type BackupConfig = {
+    backupURL: string;
+    litestreamVersion: LitestreamVersion;
+};
+export type SQLiteCatchupOptions = {
+    /** The change-log database, i.e. `changeLogFileName(replicaFile)`. */
+    changeLogFile: string;
+    readBatchRows: number;
+    barrierTimeoutMs: number;
+    /**
+     * Backstop for the change-log writer's ACK, which is what normally releases
+     * the barrier. Defaults to the catchup coordinator's interval.
+     */
+    barrierPollIntervalMs?: number | undefined;
+    /**
+     * Optional policy hook used by focused tests. Production canary selection is
+     * supplied by `sqliteChangeLogServe` and is stable by shard plus task ID.
+     * Backup subscribers are rejected before either selector is invoked.
+     */
+    shouldUse?: ((ctx: SubscriberContext) => boolean) | undefined;
+    /**
+     * Overrides the purge scheduler's guard, for tests. When absent, the
+     * service supplies the scheduler's real writer-serialized guard, or the
+     * catchup's no-op default when purging is not configured.
+     */
+    cleanupGuard?: SQLiteChangeLogCleanupGuard | undefined;
+    /**
+     * How long the change log may be unavailable before declining to serve from
+     * it is reported as a warning rather than a debug line. Defaults to
+     * {@link DEFAULT_CHANGE_LOG_UNAVAILABLE_WARN_THRESHOLD_MS}.
+     */
+    notReadyWarnThresholdMs?: number | undefined;
+};
+export type SQLiteChangeLogServeOptions = {
+    /** Stable percentage of eligible serving tasks routed to SQLite. */
+    readPercent: number;
+    /**
+     * Stable percentage of eligible serving tasks routed to a log that has not
+     * yet aged through {@link retentionMs}. Zero keeps a reseeded log on PG for
+     * that whole window; above zero it serves that share of tasks, and a
+     * reservation it cannot cover is demoted rather than held pending.
+     * Defaults to zero.
+     */
+    coldReadPercent?: number | undefined;
+    /** The warm-up window a reseeded log ages through. */
+    retentionMs: number;
+    /** Injectable for deterministic breaker tests. */
+    failureCooldownMs?: number | undefined;
+    /** Injectable for deterministic warm-up and breaker tests. */
+    now?: (() => number) | undefined;
+    /** Injectable readiness inspection for service-level tests. */
+    inspect?: (() => SQLiteChangeLogCoverage | undefined) | undefined;
+};
 export type TuningOptions = StorerOptions & {
-    flowControlConsensusPaddingSeconds: number;
+    flowControlConsensusTimeoutProportion: number;
+    flowControlSlowSubscriberGracePeriodMs?: number | undefined;
+    sqliteCatchup?: SQLiteCatchupOptions | undefined;
+    /**
+     * Supplied when `sqliteChangeLogMode != off`, i.e. this is the gate on the
+     * change-log writer. Absent, nothing writes the log.
+     */
+    sqliteChangeLogWriter?: Omit<SQLiteChangeLogWriterOptions, 'onCommit' | 'onDisabled' | 'onRebuilt'> | undefined;
+    /**
+     * Also supplied when `sqliteChangeLogMode != off`, and deliberately not
+     * gated on the read path: in `write` mode no reader ever opens, and this is
+     * the configuration the purge scheduler actually ships in. The scheduler
+     * runs on the writer's own connection, so with the writer absent (mode
+     * `off`) it is never constructed, and a leftover file on disk is left to
+     * the replicator's cleanup.
+     */
+    sqliteChangeLogPurge?: SQLiteChangeLogPurgeSchedulerOptions | undefined;
+    /**
+     * Supplied in `compare` mode and later modes.
+     * `replicaFile` enables the initialization comparison.
+     * The other fields configure sampled catchup comparisons.
+     * Both checks are advisory, and Postgres remains authoritative.
+     * The comparator also requires the writer and catchup options.
+     */
+    sqliteChangeLogCompare?: (SQLiteChangeLogCompareOptions & {
+        replicaFile: string;
+    }) | undefined;
+    /** Supplied only in `serve` mode. A zero percentage keeps every read on PG. */
+    sqliteChangeLogServe?: SQLiteChangeLogServeOptions | undefined;
 };
 /**
  * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.
  */
-export declare function initializeStreamer(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, changeDB: PostgresDB, changeSource: ChangeSource, replicationStatusPublisher: ReplicationStatusPublisher, subscriptionState: SubscriptionState, purgeLock: PurgeLock | null, autoReset: boolean, opts: TuningOptions, setTimeoutFn?: typeof setTimeout): Promise<ChangeStreamerService>;
-export type ChangeTag = ChangeStreamData[1]['tag'];
-/**
- * Internally all Downstream messages (not just commits) are given a watermark.
- * These are used for internal ordering for:
- * 1. Replaying new changes in the Storer
- * 2. Filtering old changes in the Subscriber
- *
- * However, only the watermark for `Commit` messages are exposed to
- * subscribers, as that is the only semantically correct watermark to
- * use for tracking a position in a replication stream.
- *
- * Additionally, the ChangeStreamData is eagerly stringified once, after which
- * the string is passed to the changeLog and all subscribers, eliminating
- * redundant stringification and reducing GC churn.
- */
-export type WatermarkedChange = [
-    watermark: string,
-    tag: ChangeTag,
-    json: string
-];
+export declare function initializeStreamer(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, changeDB: PostgresDB, changeSource: ChangeSource, replicationStatusPublisher: ReplicationStatusPublisher, subscriptionState: SubscriptionState, backupConfig: BackupConfig | null, purgeLock: PurgeLock | null, autoReset: boolean, opts: TuningOptions, setTimeoutFn?: typeof setTimeout): Promise<ChangeStreamerService>;
 //# sourceMappingURL=change-streamer-service.d.ts.map

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,gBAAgB,EAEtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAGrB,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C,kCAAkC,EAAE,MAAM,CAAC;CAC5C,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CA4BhC;AAID,MAAM,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM;IACjB,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,MAAM;CACb,CAAC"}
+{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAYjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAM3C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAUjF,OAAO,EACL,KAAK,qBAAqB,EAE1B,KAAK,iBAAiB,EAEvB,MAAM,sBAAsB,CAAC;AAU9B,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAEL,KAAK,6BAA6B,EACnC,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAGL,KAAK,oCAAoC,EAC1C,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAIL,KAAK,uBAAuB,EAC7B,MAAM,oCAAoC,CAAC;AAE5C,OAAO,EAEL,KAAK,4BAA4B,EAClC,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAIrB,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,iBAAiB,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,sEAAsE;IACtE,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,GAAG,SAAS,CAAC;IAC9D;;;;OAIG;IACH,YAAY,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;IACvD;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACxC,oEAAoE;IACpE,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,sDAAsD;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,8DAA8D;IAC9D,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IACjC,+DAA+D;IAC/D,OAAO,CAAC,EAAE,CAAC,MAAM,uBAAuB,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;CACnE,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5D,aAAa,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC;IACjD;;;OAGG;IACH,qBAAqB,CAAC,EAClB,IAAI,CACF,4BAA4B,EAC5B,UAAU,GAAG,YAAY,GAAG,WAAW,CACxC,GACD,SAAS,CAAC;IACd;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,oCAAoC,GAAG,SAAS,CAAC;IACxE;;;;;;OAMG;IACH,sBAAsB,CAAC,EACnB,CAAC,6BAA6B,GAAG;QAAC,WAAW,EAAE,MAAM,CAAA;KAAC,CAAC,GACvD,SAAS,CAAC;IACd,+EAA+E;IAC/E,oBAAoB,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;CAChE,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CA4BhC"}