@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.40

package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts

@@ -22,24 +22,37 @@ export type ChangeSourceUpstream = v.Infer<typeof changeSourceUpstreamSchema>;
  * Backfills are automatically started for new tables and columns in a given
  * change stream session; however, if the session is terminated before the
  * backfill completes, it must be restarted with appropriate
- * {@link BackfillRequest}s when creating a new session.
+ * {@link BackfillRequest}s in subsequent session(s), as backfill state is
+ * ephemeral, per-replication-manager, and not persisted upstream.
  *
- * The `change-streamer` is responsible for tracking any changes to the table
- * name, column names, or table metadata, and constructing a BackfillRequest
- * based on the current values (which may be different from when the
- * tables/columns were originally added).
+ * All replication subscribers track their backfill state, which includes
+ * tracking any changes to the table name, column names, or table metadata,
+ * and present that state to the `change-streamer` when starting a
+ * subscription.
+ *
+ * From there the `change-streamer` manages requesting from the change-source
+ * the superset (and minimum progressMark) of all subscriber-specified
+ * backfills. While subscribers are connected, the change-streamer also tracks
+ * this per-subscriber state in memory, so that it can properly resume
+ * backfills on a new change-stream if disconnected from upstream.
  */
 export declare const backfillRequestSchema: v.ObjectType<{
     table: v.ObjectType<Omit<{
         schema: v.Type<string>;
         name: v.Type<string>;
     }, "metadata"> & {
-        metadata: v.Type<{
-            [x: string]: import("../../../../../../shared/src/bigint-json.ts").JSONValue;
-            rowKey: Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>;
-        } | null>;
+        metadata: v.UnionType<[v.Type<null>, v.ObjectType<{
+            rowKey: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>;
+        }, v.Type<import("../../../../../../shared/src/bigint-json.ts").JSONValue>>]>;
     }, undefined>;
-    columns: v.Type<Record<string, Record<string, import("./json.ts").JSONValue | undefined>>>;
+    columns: v.Type<Record<string, {
+        id: Record<string, import("./json.ts").JSONValue | undefined>;
+        progress?: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        } | undefined;
+    }>>;
 }, undefined>;
 export type BackfillRequest = v.Infer<typeof backfillRequestSchema>;
 //# sourceMappingURL=upstream.d.ts.map

package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"upstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAQ5D,sEAAsE;AACtE,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;eAA8B,CAAC;AACtE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAE9E;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;aAOhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}
+{"version":3,"file":"upstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAS5D,sEAAsE;AACtE,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;eAA8B,CAAC;AACtE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAE9E;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;aAYhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/upstream.js

@@ -1,5 +1,5 @@
 import { valita_exports } from "../../../../../../shared/src/valita.js";
-import { backfillIDSchema, identifierSchema, tableMetadataSchema } from "./data.js";
+import { backfillIDSchema, backfillProgressMarkSchema, identifierSchema, tableMetadataSchema } from "./data.js";
 import { upstreamStatusMessageSchema } from "./status.js";
 //#region ../zero-cache/src/services/change-source/protocol/current/upstream.ts
 /** At the moment, the only upstream messages are status messages.  */
@@ -9,16 +9,26 @@ var changeSourceUpstreamSchema = upstreamStatusMessageSchema;
 * Backfills are automatically started for new tables and columns in a given
 * change stream session; however, if the session is terminated before the
 * backfill completes, it must be restarted with appropriate
-* {@link BackfillRequest}s when creating a new session.
+* {@link BackfillRequest}s in subsequent session(s), as backfill state is
+* ephemeral, per-replication-manager, and not persisted upstream.
 *
-* The `change-streamer` is responsible for tracking any changes to the table
-* name, column names, or table metadata, and constructing a BackfillRequest
-* based on the current values (which may be different from when the
-* tables/columns were originally added).
+* All replication subscribers track their backfill state, which includes
+* tracking any changes to the table name, column names, or table metadata,
+* and present that state to the `change-streamer` when starting a
+* subscription.
+*
+* From there the `change-streamer` manages requesting from the change-source
+* the superset (and minimum progressMark) of all subscriber-specified
+* backfills. While subscribers are connected, the change-streamer also tracks
+* this per-subscriber state in memory, so that it can properly resume
+* backfills on a new change-stream if disconnected from upstream.
 */
 var backfillRequestSchema = valita_exports.object({
 	table: identifierSchema.extend({ metadata: tableMetadataSchema.nullable() }),
-	columns: valita_exports.record(backfillIDSchema)
+	columns: valita_exports.record(valita_exports.object({
+		id: backfillIDSchema,
+		progress: backfillProgressMarkSchema.optional()
+	}))
 });
 //#endregion
 export { backfillRequestSchema, changeSourceUpstreamSchema };

package/out/zero-cache/src/services/change-source/protocol/current/upstream.js.map

@@ -1 +1 @@
-{"version":3,"file":"upstream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {\n  backfillIDSchema,\n  identifierSchema,\n  tableMetadataSchema,\n} from './data.ts';\nimport {upstreamStatusMessageSchema} from './status.ts';\n\n/** At the moment, the only upstream messages are status messages.  */\nexport const changeSourceUpstreamSchema = upstreamStatusMessageSchema;\nexport type ChangeSourceUpstream = v.Infer<typeof changeSourceUpstreamSchema>;\n\n/**\n * Contains the information for requesting a backfill of columns in a table.\n * Backfills are automatically started for new tables and columns in a given\n * change stream session; however, if the session is terminated before the\n * backfill completes, it must be restarted with appropriate\n * {@link BackfillRequest}s when creating a new session.\n *\n * The `change-streamer` is responsible for tracking any changes to the table\n * name, column names, or table metadata, and constructing a BackfillRequest\n * based on the current values (which may be different from when the\n * tables/columns were originally added).\n */\nexport const backfillRequestSchema = v.object({\n  table: identifierSchema.extend({\n    // The table metadata is set to null if it is never specified by the\n    // change-source.\n    metadata: tableMetadataSchema.nullable(),\n  }),\n  columns: v.record(backfillIDSchema),\n});\n\nexport type BackfillRequest = v.Infer<typeof backfillRequestSchema>;\n"],"mappings":";;;;;AASA,IAAa,6BAA6B;;;;;;;;;;;;;AAe1C,IAAa,wBAAwB,eAAE,OAAO;CAC5C,OAAO,iBAAiB,OAAO,EAG7B,UAAU,oBAAoB,SAAS,EACzC,CAAC;CACD,SAAS,eAAE,OAAO,gBAAgB;AACpC,CAAC"}
+{"version":3,"file":"upstream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {\n  backfillIDSchema,\n  backfillProgressMarkSchema,\n  identifierSchema,\n  tableMetadataSchema,\n} from './data.ts';\nimport {upstreamStatusMessageSchema} from './status.ts';\n\n/** At the moment, the only upstream messages are status messages.  */\nexport const changeSourceUpstreamSchema = upstreamStatusMessageSchema;\nexport type ChangeSourceUpstream = v.Infer<typeof changeSourceUpstreamSchema>;\n\n/**\n * Contains the information for requesting a backfill of columns in a table.\n * Backfills are automatically started for new tables and columns in a given\n * change stream session; however, if the session is terminated before the\n * backfill completes, it must be restarted with appropriate\n * {@link BackfillRequest}s in subsequent session(s), as backfill state is\n * ephemeral, per-replication-manager, and not persisted upstream.\n *\n * All replication subscribers track their backfill state, which includes\n * tracking any changes to the table name, column names, or table metadata,\n * and present that state to the `change-streamer` when starting a\n * subscription.\n *\n * From there the `change-streamer` manages requesting from the change-source\n * the superset (and minimum progressMark) of all subscriber-specified\n * backfills. While subscribers are connected, the change-streamer also tracks\n * this per-subscriber state in memory, so that it can properly resume\n * backfills on a new change-stream if disconnected from upstream.\n */\nexport const backfillRequestSchema = v.object({\n  table: identifierSchema.extend({\n    // The table metadata is set to null if it is never specified by the\n    // change-source.\n    metadata: tableMetadataSchema.nullable(),\n  }),\n  columns: v.record(\n    v.object({\n      id: backfillIDSchema,\n      progress: backfillProgressMarkSchema.optional(),\n    }),\n  ),\n});\n\nexport type BackfillRequest = v.Infer<typeof backfillRequestSchema>;\n"],"mappings":";;;;;AAUA,IAAa,6BAA6B;;;;;;;;;;;;;;;;;;;;AAsB1C,IAAa,wBAAwB,eAAE,OAAO;CAC5C,OAAO,iBAAiB,OAAO,EAG7B,UAAU,oBAAoB,SAAS,EACzC,CAAC;CACD,SAAS,eAAE,OACT,eAAE,OAAO;EACP,IAAI;EACJ,UAAU,2BAA2B,SAAS;CAChD,CAAC,CACH;AACF,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current.js

@@ -1,7 +1,7 @@
 import { __exportAll } from "../../../../../_virtual/_rolldown/runtime.js";
 import { jsonObjectSchema, jsonValueSchema } from "./current/json.js";
 import { resetRequiredSchema } from "./current/control.js";
-import { addColumnSchema, backfillCompletedSchema, backfillIDSchema, backfillSchema, beginSchema, commitSchema, createIndexSchema, createTableSchema, dataChangeSchema, deleteSchema, downloadStatusSchema, dropColumnSchema, dropIndexSchema, dropTableSchema, identifierSchema, insertSchema, isDataChange, isSchemaChange, newRelationSchema, relationSchema, renameTableSchema, rollbackSchema, rowSchema, schemaChangeSchema, tableMetadataSchema, truncateSchema, updateColumnSchema, updateSchema, updateTableMetadataSchema } from "./current/data.js";
+import { addColumnSchema, backfillCompletedSchema, backfillIDSchema, backfillProgressMarkSchema, backfillSchema, beginSchema, commitSchema, createIndexSchema, createTableSchema, dataChangeSchema, deleteSchema, downloadStatusSchema, dropColumnSchema, dropIndexSchema, dropTableSchema, identifierSchema, insertSchema, isDataChange, isSchemaChange, newRelationSchema, relationSchema, renameTableSchema, rollbackSchema, rowSchema, schemaChangeSchema, tableMetadataSchema, truncateSchema, updateColumnSchema, updateSchema, updateTableMetadataSchema } from "./current/data.js";
 import { downstreamStatusMessageSchema, downstreamStatusSchema, upstreamStatusMessageSchema } from "./current/status.js";
 import { changeStreamControlSchema, changeStreamDataSchema, changeStreamMessageSchema } from "./current/downstream.js";
 import { CHANGE_SOURCE_PATH } from "./current/path.js";
@@ -12,6 +12,7 @@ var current_exports = /* @__PURE__ */ __exportAll({
 	addColumnSchema: () => addColumnSchema,
 	backfillCompletedSchema: () => backfillCompletedSchema,
 	backfillIDSchema: () => backfillIDSchema,
+	backfillProgressMarkSchema: () => backfillProgressMarkSchema,
 	backfillRequestSchema: () => backfillRequestSchema,
 	backfillSchema: () => backfillSchema,
 	beginSchema: () => beginSchema,

package/out/zero-cache/src/services/change-streamer/backfill-state-tracker.d.ts

@@ -0,0 +1,72 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { Subscription } from '../../types/subscription.ts';
+import type { ChangeStreamData } from '../change-source/protocol/current/downstream.ts';
+import type { BackfillRequest } from '../change-source/protocol/current/upstream.ts';
+import type { PreSerializedBatch } from './broadcast.ts';
+import type { SubscriberContext } from './change-streamer.ts';
+import type { Subscriber, SubscriberOptions } from './subscriber.ts';
+export type SubscriberBackfillOptions = Pick<SubscriberOptions, 'backfills' | 'onAligned' | 'onBackfillIgnored'>;
+/**
+ * Tracks the pending backfills (and their progress) of the change log, the
+ * subscribers, and the current change stream, in order to determine:
+ *
+ * * the BackfillRequests with which to start a change stream, i.e. the
+ *   superset of the pending backfills of the change log and all aligned
+ *   subscribers, starting from the earliest progress, and
+ * * whether (and why) the change stream must be restarted, i.e. when a
+ *   newly aligned subscriber's pending backfills are not covered by the
+ *   current stream.
+ *
+ * The change-streamer calls {@link startStream()} whenever it starts a change
+ * stream, {@link track()} for every change that it forwards, and restarts
+ * the stream at the next transaction boundary whenever a
+ * {@link restartReason} is set.
+ */
+export declare class BackfillStateTracker {
+    #private;
+    constructor(lc: LogContext);
+    /**
+     * The reason for which the change stream must be restarted, or `null` if
+     * it need not be. The restart should happen at the next transaction
+     * boundary, i.e. upon the next `commit`, or `status` message outside of a
+     * transaction.
+     */
+    get restartReason(): string | null;
+    /**
+     * Computes the BackfillRequests with which to start a new change stream:
+     * the superset of the pending backfills of the change log and of all
+     * aligned subscribers, all of which are consistent with the stream's
+     * starting watermark since streams are (re)started at transaction
+     * boundaries.
+     *
+     * @param logBackfillRequests The BackfillRequests from the change log's
+     *        cookie jar, consistent with the stream's starting watermark.
+     */
+    startStream(logBackfillRequests: BackfillRequest[]): BackfillRequest[];
+    /**
+     * Tracks a change that is forwarded to subscribers. This must be called
+     * (synchronously) before the change is forwarded, so that the stream's
+     * state is never ahead of that of aligned subscribers.
+     */
+    track(change: ChangeStreamData): void;
+    /**
+     * Requests a restart of the change stream (at the next transaction
+     * boundary), e.g. to start it with BackfillRequests that it was not
+     * started with.
+     */
+    requestRestart(reason: string): void;
+    /** Logs and records a (planned) restart for the {@link restartReason}. */
+    recordRestart(): void;
+    /**
+     * Returns the backfill-related {@link SubscriberOptions} for a new
+     * subscriber, which must subsequently be passed to {@link register()}.
+     */
+    subscriberOptions(lc: LogContext, ctx: SubscriberContext): SubscriberBackfillOptions;
+    /**
+     * Registers a subscriber (created with {@link subscriberOptions()}) until
+     * its `downstream` is closed. This is a no-op for subscribers that do not
+     * report their pending backfills.
+     */
+    register(subscriber: Subscriber, downstream: Subscription<string | PreSerializedBatch>): void;
+}
+//# sourceMappingURL=backfill-state-tracker.d.ts.map

package/out/zero-cache/src/services/change-streamer/backfill-state-tracker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"backfill-state-tracker.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backfill-state-tracker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAK9D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AAEnF,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAC,UAAU,EAAE,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAEnE,MAAM,MAAM,yBAAyB,GAAG,IAAI,CAC1C,iBAAiB,EACjB,WAAW,GAAG,WAAW,GAAG,mBAAmB,CAChD,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,oBAAoB;;IA6C/B,YAAY,EAAE,EAAE,UAAU,EAEzB;IAED;;;;;OAKG;IACH,IAAI,aAAa,IAAI,MAAM,GAAG,IAAI,CAEjC;IAED;;;;;;;;;OASG;IACH,WAAW,CAAC,mBAAmB,EAAE,eAAe,EAAE,GAAG,eAAe,EAAE,CAyBrE;IAED;;;;OAIG;IACH,KAAK,CAAC,MAAM,EAAE,gBAAgB,QAG7B;IAED;;;;OAIG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,QAE5B;IAED,0EAA0E;IAC1E,aAAa,SAGZ;IAED;;;OAGG;IACH,iBAAiB,CACf,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,iBAAiB,GACrB,yBAAyB,CAqB3B;IAED;;;;OAIG;IACH,QAAQ,CACN,UAAU,EAAE,UAAU,EACtB,UAAU,EAAE,YAAY,CAAC,MAAM,GAAG,kBAAkB,CAAC,QAMtD;CAsBF"}

package/out/zero-cache/src/services/change-streamer/backfill-state-tracker.js

@@ -0,0 +1,162 @@
+import { must } from "../../../../shared/src/must.js";
+import { getOrCreateCounter } from "../../observability/metrics.js";
+import { getSuperset, withResumePoint } from "../change-source/protocol/backfill-progress.js";
+import { BackfillState } from "./backfill-state.js";
+//#region ../zero-cache/src/services/change-streamer/backfill-state-tracker.ts
+/**
+* Tracks the pending backfills (and their progress) of the change log, the
+* subscribers, and the current change stream, in order to determine:
+*
+* * the BackfillRequests with which to start a change stream, i.e. the
+*   superset of the pending backfills of the change log and all aligned
+*   subscribers, starting from the earliest progress, and
+* * whether (and why) the change stream must be restarted, i.e. when a
+*   newly aligned subscriber's pending backfills are not covered by the
+*   current stream.
+*
+* The change-streamer calls {@link startStream()} whenever it starts a change
+* stream, {@link track()} for every change that it forwards, and restarts
+* the stream at the next transaction boundary whenever a
+* {@link restartReason} is set.
+*/
+var BackfillStateTracker = class {
+	#lc;
+	/**
+	* The pending backfills of the change log itself, i.e. those of a
+	* (hypothetical) subscriber that has applied every change in the log.
+	* Whenever a stream is started, it is reset from the change log's cookie
+	* jar (which is consistent with the stream's starting watermark), carrying
+	* over the progress tracked in memory (as the cookie jar is
+	* progress-agnostic). It stands in for subscribers that do not report their
+	* pending backfills (i.e. protocol < v8), and ensures that backfills that
+	* are pending in the change log continue to be requested.
+	*/
+	#logBackfills = new BackfillState();
+	/**
+	* The backfill progress of the current change stream (i.e. "session"),
+	* seeded from the BackfillRequests with which the stream was started.
+	* This determines whether a (newly aligned) subscriber's pending backfills
+	* are covered by the stream.
+	*/
+	#sessionBackfills = new BackfillState();
+	/** Subscribers that report their pending backfills (protocol v8+). */
+	#subscribers = /* @__PURE__ */ new Set();
+	/**
+	* Set (to the reason) when the change stream must be restarted with
+	* new BackfillRequests.
+	*/
+	#restartReason = null;
+	#restarts = getOrCreateCounter("replication", "backfill_stream_restarts", "Count of change stream restarts initiated to rewind backfills for subscribers whose pending backfills were not covered by the stream");
+	#unexpectedIgnores = getOrCreateCounter("replication", "backfill_unexpected_ignores", "Count of backfill messages unexpectedly ignored (for some columns) by an aligned subscriber. The expected count is 0.");
+	constructor(lc) {
+		this.#lc = lc;
+	}
+	/**
+	* The reason for which the change stream must be restarted, or `null` if
+	* it need not be. The restart should happen at the next transaction
+	* boundary, i.e. upon the next `commit`, or `status` message outside of a
+	* transaction.
+	*/
+	get restartReason() {
+		return this.#restartReason;
+	}
+	/**
+	* Computes the BackfillRequests with which to start a new change stream:
+	* the superset of the pending backfills of the change log and of all
+	* aligned subscribers, all of which are consistent with the stream's
+	* starting watermark since streams are (re)started at transaction
+	* boundaries.
+	*
+	* @param logBackfillRequests The BackfillRequests from the change log's
+	*        cookie jar, consistent with the stream's starting watermark.
+	*/
+	startStream(logBackfillRequests) {
+		this.#logBackfills = new BackfillState(this.#logBackfills.withProgress(logBackfillRequests));
+		const aligned = [...this.#subscribers].filter((s) => s.aligned);
+		const requests = getSuperset(this.#logBackfills.requests(), ...aligned.map((s) => must(s.backfills).requests())).map(withResumePoint);
+		this.#sessionBackfills = new BackfillState(requests);
+		this.#restartReason = null;
+		if (requests.length) this.#lc.info?.(`starting change stream with ${requests.length} backfill requests (from the change log and ${aligned.length} subscribers)`, { requests });
+		return requests;
+	}
+	/**
+	* Tracks a change that is forwarded to subscribers. This must be called
+	* (synchronously) before the change is forwarded, so that the stream's
+	* state is never ahead of that of aligned subscribers.
+	*/
+	track(change) {
+		this.#sessionBackfills.apply(change);
+		this.#logBackfills.apply(change);
+	}
+	/**
+	* Requests a restart of the change stream (at the next transaction
+	* boundary), e.g. to start it with BackfillRequests that it was not
+	* started with.
+	*/
+	requestRestart(reason) {
+		this.#restartReason ??= reason;
+	}
+	/** Logs and records a (planned) restart for the {@link restartReason}. */
+	recordRestart() {
+		this.#lc.info?.(`restarting change stream: ${this.#restartReason}`);
+		this.#restarts.add(1);
+	}
+	/**
+	* Returns the backfill-related {@link SubscriberOptions} for a new
+	* subscriber, which must subsequently be passed to {@link register()}.
+	*/
+	subscriberOptions(lc, ctx) {
+		const backfills = newBackfillState(lc, ctx);
+		if (!backfills) return {};
+		return {
+			backfills,
+			onAligned: (subscriber) => {
+				if (this.#subscribers.has(subscriber)) this.#checkCoverage(subscriber);
+			},
+			onBackfillIgnored: (columns) => {
+				if (this.#restartReason === null) {
+					lc.warn?.(`aligned subscriber ignored backfill data for ${columns.join(", ")}`);
+					this.#unexpectedIgnores.add(1);
+				}
+			}
+		};
+	}
+	/**
+	* Registers a subscriber (created with {@link subscriberOptions()}) until
+	* its `downstream` is closed. This is a no-op for subscribers that do not
+	* report their pending backfills.
+	*/
+	register(subscriber, downstream) {
+		if (subscriber.backfills) {
+			this.#subscribers.add(subscriber);
+			downstream.addCloseHandler(() => this.#subscribers.delete(subscriber));
+		}
+	}
+	/**
+	* Checks whether the pending backfills of a newly aligned subscriber are
+	* covered by the current stream, requesting a restart of the stream if not.
+	*
+	* This can be checked as soon as the subscriber is aligned (even in the
+	* middle of a transaction), since both the subscriber's and the stream's
+	* BackfillStates compare their committed state, and the stream's state is
+	* never updated with a change before the change is forwarded to (aligned)
+	* subscribers. The restart itself happens at the next transaction boundary.
+	*/
+	#checkCoverage(subscriber) {
+		const uncovered = must(subscriber.backfills).uncovered(this.#sessionBackfills);
+		if (uncovered.length) this.#restartReason ??= `subscriber ${subscriber.id} needs backfills that are not covered by the current stream: ${uncovered.join(", ")}`;
+	}
+};
+function newBackfillState(lc, ctx) {
+	if (ctx.backfills === void 0) return;
+	try {
+		return new BackfillState(ctx.backfills);
+	} catch (e) {
+		lc.warn?.(`invalid backfills from subscriber ${ctx.id}`, e);
+		return;
+	}
+}
+//#endregion
+export { BackfillStateTracker };
+
+//# sourceMappingURL=backfill-state-tracker.js.map

package/out/zero-cache/src/services/change-streamer/backfill-state-tracker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"backfill-state-tracker.js","names":["#lc","#subscribers","#restarts","#unexpectedIgnores","#restartReason","#logBackfills","#sessionBackfills","#checkCoverage"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backfill-state-tracker.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {must} from '../../../../shared/src/must.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport {\n  getSuperset,\n  withResumePoint,\n} from '../change-source/protocol/backfill-progress.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport {BackfillState} from './backfill-state.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {SubscriberContext} from './change-streamer.ts';\nimport type {Subscriber, SubscriberOptions} from './subscriber.ts';\n\nexport type SubscriberBackfillOptions = Pick<\n  SubscriberOptions,\n  'backfills' | 'onAligned' | 'onBackfillIgnored'\n>;\n\n/**\n * Tracks the pending backfills (and their progress) of the change log, the\n * subscribers, and the current change stream, in order to determine:\n *\n * * the BackfillRequests with which to start a change stream, i.e. the\n *   superset of the pending backfills of the change log and all aligned\n *   subscribers, starting from the earliest progress, and\n * * whether (and why) the change stream must be restarted, i.e. when a\n *   newly aligned subscriber's pending backfills are not covered by the\n *   current stream.\n *\n * The change-streamer calls {@link startStream()} whenever it starts a change\n * stream, {@link track()} for every change that it forwards, and restarts\n * the stream at the next transaction boundary whenever a\n * {@link restartReason} is set.\n */\nexport class BackfillStateTracker {\n  readonly #lc: LogContext;\n\n  /**\n   * The pending backfills of the change log itself, i.e. those of a\n   * (hypothetical) subscriber that has applied every change in the log.\n   * Whenever a stream is started, it is reset from the change log's cookie\n   * jar (which is consistent with the stream's starting watermark), carrying\n   * over the progress tracked in memory (as the cookie jar is\n   * progress-agnostic). It stands in for subscribers that do not report their\n   * pending backfills (i.e. protocol < v8), and ensures that backfills that\n   * are pending in the change log continue to be requested.\n   */\n  #logBackfills = new BackfillState();\n\n  /**\n   * The backfill progress of the current change stream (i.e. \"session\"),\n   * seeded from the BackfillRequests with which the stream was started.\n   * This determines whether a (newly aligned) subscriber's pending backfills\n   * are covered by the stream.\n   */\n  #sessionBackfills = new BackfillState();\n\n  /** Subscribers that report their pending backfills (protocol v8+). */\n  readonly #subscribers = new Set<Subscriber>();\n\n  /**\n   * Set (to the reason) when the change stream must be restarted with\n   * new BackfillRequests.\n   */\n  #restartReason: string | null = null;\n\n  readonly #restarts = getOrCreateCounter(\n    'replication',\n    'backfill_stream_restarts',\n    'Count of change stream restarts initiated to rewind backfills for ' +\n      'subscribers whose pending backfills were not covered by the stream',\n  );\n  readonly #unexpectedIgnores = getOrCreateCounter(\n    'replication',\n    'backfill_unexpected_ignores',\n    'Count of backfill messages unexpectedly ignored (for some columns) by ' +\n      'an aligned subscriber. The expected count is 0.',\n  );\n\n  constructor(lc: LogContext) {\n    this.#lc = lc;\n  }\n\n  /**\n   * The reason for which the change stream must be restarted, or `null` if\n   * it need not be. The restart should happen at the next transaction\n   * boundary, i.e. upon the next `commit`, or `status` message outside of a\n   * transaction.\n   */\n  get restartReason(): string | null {\n    return this.#restartReason;\n  }\n\n  /**\n   * Computes the BackfillRequests with which to start a new change stream:\n   * the superset of the pending backfills of the change log and of all\n   * aligned subscribers, all of which are consistent with the stream's\n   * starting watermark since streams are (re)started at transaction\n   * boundaries.\n   *\n   * @param logBackfillRequests The BackfillRequests from the change log's\n   *        cookie jar, consistent with the stream's starting watermark.\n   */\n  startStream(logBackfillRequests: BackfillRequest[]): BackfillRequest[] {\n    // The cookie jar can be behind #logBackfills (e.g. if the storer failed\n    // to persist a forwarded commit), in which case the new stream\n    // re-delivers those changes.\n    this.#logBackfills = new BackfillState(\n      this.#logBackfills.withProgress(logBackfillRequests),\n    );\n\n    const aligned = [...this.#subscribers].filter(s => s.aligned);\n    const requests = getSuperset(\n      this.#logBackfills.requests(),\n      ...aligned.map(s => must(s.backfills).requests()),\n    ).map(withResumePoint);\n\n    this.#sessionBackfills = new BackfillState(requests);\n    // Aligned subscribers are accounted for in the requests.\n    this.#restartReason = null;\n    if (requests.length) {\n      this.#lc.info?.(\n        `starting change stream with ${requests.length} backfill requests ` +\n          `(from the change log and ${aligned.length} subscribers)`,\n        {requests},\n      );\n    }\n    return requests;\n  }\n\n  /**\n   * Tracks a change that is forwarded to subscribers. This must be called\n   * (synchronously) before the change is forwarded, so that the stream's\n   * state is never ahead of that of aligned subscribers.\n   */\n  track(change: ChangeStreamData) {\n    this.#sessionBackfills.apply(change);\n    this.#logBackfills.apply(change);\n  }\n\n  /**\n   * Requests a restart of the change stream (at the next transaction\n   * boundary), e.g. to start it with BackfillRequests that it was not\n   * started with.\n   */\n  requestRestart(reason: string) {\n    this.#restartReason ??= reason;\n  }\n\n  /** Logs and records a (planned) restart for the {@link restartReason}. */\n  recordRestart() {\n    this.#lc.info?.(`restarting change stream: ${this.#restartReason}`);\n    this.#restarts.add(1);\n  }\n\n  /**\n   * Returns the backfill-related {@link SubscriberOptions} for a new\n   * subscriber, which must subsequently be passed to {@link register()}.\n   */\n  subscriberOptions(\n    lc: LogContext,\n    ctx: SubscriberContext,\n  ): SubscriberBackfillOptions {\n    const backfills = newBackfillState(lc, ctx);\n    if (!backfills) {\n      return {};\n    }\n    return {\n      backfills,\n      onAligned: subscriber => {\n        if (this.#subscribers.has(subscriber)) {\n          this.#checkCoverage(subscriber);\n        }\n      },\n      onBackfillIgnored: columns => {\n        if (this.#restartReason === null) {\n          lc.warn?.(\n            `aligned subscriber ignored backfill data for ${columns.join(', ')}`,\n          );\n          this.#unexpectedIgnores.add(1);\n        }\n      },\n    };\n  }\n\n  /**\n   * Registers a subscriber (created with {@link subscriberOptions()}) until\n   * its `downstream` is closed. This is a no-op for subscribers that do not\n   * report their pending backfills.\n   */\n  register(\n    subscriber: Subscriber,\n    downstream: Subscription<string | PreSerializedBatch>,\n  ) {\n    if (subscriber.backfills) {\n      this.#subscribers.add(subscriber);\n      downstream.addCloseHandler(() => this.#subscribers.delete(subscriber));\n    }\n  }\n\n  /**\n   * Checks whether the pending backfills of a newly aligned subscriber are\n   * covered by the current stream, requesting a restart of the stream if not.\n   *\n   * This can be checked as soon as the subscriber is aligned (even in the\n   * middle of a transaction), since both the subscriber's and the stream's\n   * BackfillStates compare their committed state, and the stream's state is\n   * never updated with a change before the change is forwarded to (aligned)\n   * subscribers. The restart itself happens at the next transaction boundary.\n   */\n  #checkCoverage(subscriber: Subscriber) {\n    const uncovered = must(subscriber.backfills).uncovered(\n      this.#sessionBackfills,\n    );\n    if (uncovered.length) {\n      this.#restartReason ??=\n        `subscriber ${subscriber.id} needs backfills that are not ` +\n        `covered by the current stream: ${uncovered.join(', ')}`;\n    }\n  }\n}\n\nfunction newBackfillState(\n  lc: LogContext,\n  ctx: SubscriberContext,\n): BackfillState | undefined {\n  if (ctx.backfills === undefined) {\n    return undefined; // protocol < v8\n  }\n  try {\n    return new BackfillState(ctx.backfills);\n  } catch (e) {\n    lc.warn?.(`invalid backfills from subscriber ${ctx.id}`, e);\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAoCA,IAAa,uBAAb,MAAkC;CAChC;;;;;;;;;;;CAYA,gBAAgB,IAAI,cAAc;;;;;;;CAQlC,oBAAoB,IAAI,cAAc;;CAGtC,+BAAwB,IAAI,IAAgB;;;;;CAM5C,iBAAgC;CAEhC,YAAqB,mBACnB,eACA,4BACA,sIAEF;CACA,qBAA8B,mBAC5B,eACA,+BACA,uHAEF;CAEA,YAAY,IAAgB;EAC1B,KAAKA,MAAM;CACb;;;;;;;CAQA,IAAI,gBAA+B;EACjC,OAAO,KAAKI;CACd;;;;;;;;;;;CAYA,YAAY,qBAA2D;EAIrE,KAAKC,gBAAgB,IAAI,cACvB,KAAKA,cAAc,aAAa,mBAAmB,CACrD;EAEA,MAAM,UAAU,CAAC,GAAG,KAAKJ,YAAY,EAAE,QAAO,MAAK,EAAE,OAAO;EAC5D,MAAM,WAAW,YACf,KAAKI,cAAc,SAAS,GAC5B,GAAG,QAAQ,KAAI,MAAK,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAClD,EAAE,IAAI,eAAe;EAErB,KAAKC,oBAAoB,IAAI,cAAc,QAAQ;EAEnD,KAAKF,iBAAiB;EACtB,IAAI,SAAS,QACX,KAAKJ,IAAI,OACP,+BAA+B,SAAS,OAAO,8CACjB,QAAQ,OAAO,gBAC7C,EAAC,SAAQ,CACX;EAEF,OAAO;CACT;;;;;;CAOA,MAAM,QAA0B;EAC9B,KAAKM,kBAAkB,MAAM,MAAM;EACnC,KAAKD,cAAc,MAAM,MAAM;CACjC;;;;;;CAOA,eAAe,QAAgB;EAC7B,KAAKD,mBAAmB;CAC1B;;CAGA,gBAAgB;EACd,KAAKJ,IAAI,OAAO,6BAA6B,KAAKI,gBAAgB;EAClE,KAAKF,UAAU,IAAI,CAAC;CACtB;;;;;CAMA,kBACE,IACA,KAC2B;EAC3B,MAAM,YAAY,iBAAiB,IAAI,GAAG;EAC1C,IAAI,CAAC,WACH,OAAO,CAAC;EAEV,OAAO;GACL;GACA,YAAW,eAAc;IACvB,IAAI,KAAKD,aAAa,IAAI,UAAU,GAClC,KAAKM,eAAe,UAAU;GAElC;GACA,oBAAmB,YAAW;IAC5B,IAAI,KAAKH,mBAAmB,MAAM;KAChC,GAAG,OACD,gDAAgD,QAAQ,KAAK,IAAI,GACnE;KACA,KAAKD,mBAAmB,IAAI,CAAC;IAC/B;GACF;EACF;CACF;;;;;;CAOA,SACE,YACA,YACA;EACA,IAAI,WAAW,WAAW;GACxB,KAAKF,aAAa,IAAI,UAAU;GAChC,WAAW,sBAAsB,KAAKA,aAAa,OAAO,UAAU,CAAC;EACvE;CACF;;;;;;;;;;;CAYA,eAAe,YAAwB;EACrC,MAAM,YAAY,KAAK,WAAW,SAAS,EAAE,UAC3C,KAAKK,iBACP;EACA,IAAI,UAAU,QACZ,KAAKF,mBACH,cAAc,WAAW,GAAG,+DACM,UAAU,KAAK,IAAI;CAE3D;AACF;AAEA,SAAS,iBACP,IACA,KAC2B;CAC3B,IAAI,IAAI,cAAc,KAAA,GACpB;CAEF,IAAI;EACF,OAAO,IAAI,cAAc,IAAI,SAAS;CACxC,SAAS,GAAG;EACV,GAAG,OAAO,qCAAqC,IAAI,MAAM,CAAC;EAC1D;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/backfill-state.d.ts

@@ -0,0 +1,50 @@
+import type { BackfillRequest, ChangeStreamData } from '../change-source/protocol/current.ts';
+import type { WatermarkedChange } from './change-streamer.ts';
+/**
+ * Tracks the state of the pending backfills (and their progress) of a party
+ * in the replication stream (e.g. a subscriber, or the change-log itself), by
+ * applying the same policies that a subscriber applies to its replica:
+ * schema changes are folded with the {@link cookieOps} shared by the
+ * cookie jars, and `backfill` / `backfill-completed` messages are accepted
+ * or ignored with the continuity policy of {@link acceptBackfill}.
+ *
+ * Changes are applied transactionally: the changes of a transaction are
+ * discarded if it is rolled back, and {@link requests()} only reflects
+ * committed transactions.
+ */
+export declare class BackfillState {
+    #private;
+    constructor(requests?: readonly BackfillRequest[]);
+    /** The (committed) pending backfills. */
+    requests(): BackfillRequest[];
+    /**
+     * Returns the `requests` with the (committed) progress of this state
+     * applied to the columns that are pending in both, i.e. same table, column,
+     * and backfill ID. Other columns have no progress (i.e. from scratch).
+     */
+    withProgress(requests: readonly BackfillRequest[]): BackfillRequest[];
+    /** Whether there are no (committed) pending backfills. */
+    get empty(): boolean;
+    /**
+     * Returns the columns (as `schema.table.column` strings) with (committed)
+     * pending backfills that are not covered by the (committed) progress of the
+     * `stream`, i.e. columns for which the `stream` will not deliver all of the
+     * necessary data.
+     */
+    uncovered(stream: BackfillState): string[];
+    /**
+     * Applies a serialized change, parsing it only if it is relevant.
+     *
+     * @returns The columns (as `schema.table.column` strings) of a backfill
+     *          message that were ignored due to the continuity policy.
+     */
+    applySerialized([, tag, json]: WatermarkedChange): string[];
+    /**
+     * Applies a change.
+     *
+     * @returns The columns (as `schema.table.column` strings) of a backfill
+     *          message that were ignored due to the continuity policy.
+     */
+    apply(change: ChangeStreamData): string[];
+}
+//# sourceMappingURL=backfill-state.d.ts.map

package/out/zero-cache/src/services/change-streamer/backfill-state.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"backfill-state.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backfill-state.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EACV,eAAe,EACf,gBAAgB,EAIjB,MAAM,sCAAsC,CAAC;AAG9C,OAAO,KAAK,EAAY,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AA8CvE;;;;;;;;;;;GAWG;AACH,qBAAa,aAAa;;IAIxB,YAAY,QAAQ,GAAE,SAAS,eAAe,EAAO,EASpD;IAMD,yCAAyC;IACzC,QAAQ,IAAI,eAAe,EAAE,CAE5B;IAED;;;;OAIG;IACH,YAAY,CAAC,QAAQ,EAAE,SAAS,eAAe,EAAE,GAAG,eAAe,EAAE,CAmBpE;IAED,0DAA0D;IAC1D,IAAI,KAAK,IAAI,OAAO,CAEnB;IAED;;;;;OAKG;IACH,SAAS,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,EAAE,CAazC;IAED;;;;;OAKG;IACH,eAAe,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,iBAAiB,GAAG,MAAM,EAAE,CAgB1D;IAED;;;;;OAKG;IACH,KAAK,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,EAAE,CA4DxC;CA6HF"}