@rocicorp/zero 1.10.0-canary.0 → 1.10.0-canary.2

package/out/zero-cache/src/services/replicator/change-log-stream-writer.js

@@ -0,0 +1,150 @@
+import { isSchemaChange } from "../change-source/protocol/current/data.js";
+import { ChangeLogCookieWriter } from "./change-log-cookies.js";
+import { CHANGE_LOG_STREAM_TABLE, estimateChangeLogStreamRowBytes } from "./change-log-db.js";
+import { extractChangeSubstring } from "../change-streamer/change-log-codec.js";
+import { ChangeLogTransactionHasher } from "../change-streamer/change-log-transaction-hash.js";
+//#region ../zero-cache/src/services/replicator/change-log-stream-writer.ts
+var ChangeLogStreamInvariantError = class extends Error {
+	name = "ChangeLogStreamInvariantError";
+};
+function assertInvariant(condition, message) {
+	if (!condition) throw new ChangeLogStreamInvariantError(message);
+}
+/**
+* Appends the canonical downstream stream to the change-log database, which is
+* a separate file from the replica, and folds its schema changes into the log's
+* cookie jar (`change-log-cookies.ts`) as it goes.
+*
+* This class owns the change log's transaction, and its transaction alone. It is
+* driven from the change-streamer's stream loop, which commits it before
+* forwarding a transaction's `commit` message; see `SQLiteChangeLogWriter` in
+* `change-streamer/sqlite-change-log-writer.ts` for the ordering that makes a
+* hole unreachable, and for the fail-soft policy that wraps every call here.
+*
+* Because the cookies are written in that same transaction, they are atomic with
+* the head and {@link rollback} discards them with the rows they were folded
+* from. Neither needs any handling of its own.
+*/
+var ChangeLogStreamWriter = class {
+	#db;
+	#insertChange;
+	#insertCommit;
+	#cookies;
+	#watermark;
+	#pos = 0;
+	#estimatedBytes = 0;
+	#hasher;
+	#cookieMutations = [];
+	constructor(db) {
+		this.#db = db;
+		this.#insertChange = db.db.prepare(`
+      INSERT INTO "${CHANGE_LOG_STREAM_TABLE}"
+        ("watermark", "pos", "tag", "estimatedBytes", "change")
+        VALUES (?, ?, ?, ?, ?)
+    `);
+		this.#insertCommit = db.db.prepare(`
+      INSERT INTO "${CHANGE_LOG_STREAM_TABLE}"
+        ("watermark", "pos", "tag", "estimatedBytes", "change", "precommit",
+         "writeTimeMs")
+        VALUES (?, ?, 'commit', ?, ?, ?, ?)
+    `);
+		this.#cookies = new ChangeLogCookieWriter(db.db);
+	}
+	begin(watermark, json) {
+		assertInvariant(this.#watermark === void 0, `change-log stream transaction already open at ${this.#watermark}`);
+		this.#db.beginImmediate();
+		this.#watermark = watermark;
+		this.#pos = 0;
+		const change = extractChangeSubstring(json, "begin");
+		const estimatedBytes = estimateChangeLogStreamRowBytes(watermark, "begin", change);
+		this.#insertChange.run(watermark, this.#pos, "begin", estimatedBytes, change);
+		this.#hasher = new ChangeLogTransactionHasher();
+		this.#hasher.add({
+			watermark,
+			pos: this.#pos,
+			tag: "begin",
+			change,
+			precommit: null
+		});
+		this.#estimatedBytes = estimatedBytes;
+	}
+	/**
+	* Appends one data or schema change. The parsed message is taken rather than
+	* just its tag because a schema change also mutates the cookie jar, and the
+	* caller already holds it — so the fold costs no re-parse.
+	*
+	* The cookie statements run here, i.e. inside this transaction and in stream
+	* order relative to the rows they were folded from. The Postgres change log
+	* has to flush its buffered batch to get the same ordering
+	* (`Storer.#processQueue()`); SQLite gets it from statement order.
+	*/
+	append(json, change) {
+		const watermark = this.#requireWatermark();
+		const { tag } = change;
+		const stored = extractChangeSubstring(json, tag);
+		const estimatedBytes = estimateChangeLogStreamRowBytes(watermark, tag, stored);
+		this.#insertChange.run(watermark, ++this.#pos, tag, estimatedBytes, stored);
+		this.#requireHasher().add({
+			watermark,
+			pos: this.#pos,
+			tag,
+			change: stored,
+			precommit: null
+		});
+		this.#estimatedBytes += estimatedBytes;
+		if (isSchemaChange(change)) for (const op of this.#cookies.apply(change)) this.#cookieMutations.push(op.op);
+	}
+	commit(watermark, json, writeTimeMs) {
+		const precommit = this.#requireWatermark();
+		assertInvariant(watermark === precommit, `change-log stream commit ${watermark} does not match begin ${precommit}`);
+		const change = extractChangeSubstring(json, "commit");
+		const estimatedBytes = estimateChangeLogStreamRowBytes(watermark, "commit", change, precommit, true);
+		this.#insertCommit.run(watermark, ++this.#pos, estimatedBytes, change, precommit, writeTimeMs);
+		const hasher = this.#requireHasher();
+		hasher.add({
+			watermark,
+			pos: this.#pos,
+			tag: "commit",
+			change,
+			precommit
+		});
+		const stats = {
+			rows: this.#pos + 1,
+			estimatedBytes: this.#estimatedBytes + estimatedBytes,
+			hash: hasher.digest(),
+			cookieMutations: this.#cookieMutations
+		};
+		this.#db.commit();
+		this.#reset();
+		return stats;
+	}
+	/**
+	* Discards the open transaction, if any. Safe to call when none is open — the
+	* caller rolls back on an interrupted change stream, which can arrive between
+	* transactions as well as inside one.
+	*/
+	rollback() {
+		const { inTransaction } = this.#db.db;
+		this.#reset();
+		if (inTransaction) this.#db.rollback();
+	}
+	#reset() {
+		this.#watermark = void 0;
+		this.#pos = 0;
+		this.#estimatedBytes = 0;
+		this.#hasher = void 0;
+		this.#cookieMutations = [];
+	}
+	#requireWatermark() {
+		assertInvariant(this.#watermark !== void 0, "change-log stream message received outside of a transaction");
+		return this.#watermark;
+	}
+	#requireHasher() {
+		assertInvariant(this.#hasher !== void 0, "change-log stream hash received outside of a transaction");
+		return this.#hasher;
+	}
+};
+//#endregion
+export { ChangeLogStreamWriter };
+
+//# sourceMappingURL=change-log-stream-writer.js.map

package/out/zero-cache/src/services/replicator/change-log-stream-writer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-log-stream-writer.js","names":["#db","#insertChange","#insertCommit","#cookies","#watermark","#pos","#hasher","#estimatedBytes","#requireWatermark","#requireHasher","#cookieMutations","#reset"],"sources":["../../../../../../zero-cache/src/services/replicator/change-log-stream-writer.ts"],"sourcesContent":["import type {Statement} from '../../../../zqlite/src/db.ts';\nimport type {StatementRunner} from '../../db/statements.ts';\nimport {\n  isSchemaChange,\n  type DataOrSchemaChange,\n} from '../change-source/protocol/current/data.ts';\nimport {extractChangeSubstring} from '../change-streamer/change-log-codec.ts';\nimport {ChangeLogTransactionHasher} from '../change-streamer/change-log-transaction-hash.ts';\nimport {ChangeLogCookieWriter, type CookieOpTag} from './change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  estimateChangeLogStreamRowBytes,\n} from './change-log-db.ts';\n\nexport type ChangeLogStreamTransactionStats = {\n  readonly rows: number;\n  readonly estimatedBytes: number;\n  readonly hash: string;\n  /**\n   * The cookie mutations this transaction's schema changes folded in, in stream\n   * order. Empty for the overwhelming majority of transactions.\n   */\n  readonly cookieMutations: readonly CookieOpTag[];\n};\n\nexport class ChangeLogStreamInvariantError extends Error {\n  override readonly name = 'ChangeLogStreamInvariantError';\n}\n\nfunction assertInvariant(\n  condition: unknown,\n  message: string,\n): asserts condition {\n  if (!condition) {\n    throw new ChangeLogStreamInvariantError(message);\n  }\n}\n\n/**\n * Appends the canonical downstream stream to the change-log database, which is\n * a separate file from the replica, and folds its schema changes into the log's\n * cookie jar (`change-log-cookies.ts`) as it goes.\n *\n * This class owns the change log's transaction, and its transaction alone. It is\n * driven from the change-streamer's stream loop, which commits it before\n * forwarding a transaction's `commit` message; see `SQLiteChangeLogWriter` in\n * `change-streamer/sqlite-change-log-writer.ts` for the ordering that makes a\n * hole unreachable, and for the fail-soft policy that wraps every call here.\n *\n * Because the cookies are written in that same transaction, they are atomic with\n * the head and {@link rollback} discards them with the rows they were folded\n * from. Neither needs any handling of its own.\n */\nexport class ChangeLogStreamWriter {\n  readonly #db: StatementRunner;\n  readonly #insertChange: Statement;\n  readonly #insertCommit: Statement;\n  readonly #cookies: ChangeLogCookieWriter;\n\n  #watermark: string | undefined;\n  #pos = 0;\n  #estimatedBytes = 0;\n  #hasher: ChangeLogTransactionHasher | undefined;\n  #cookieMutations: CookieOpTag[] = [];\n\n  constructor(db: StatementRunner) {\n    this.#db = db;\n    this.#insertChange = db.db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n        (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\")\n        VALUES (?, ?, ?, ?, ?)\n    `);\n    this.#insertCommit = db.db.prepare(/*sql*/ `\n      INSERT INTO \"${CHANGE_LOG_STREAM_TABLE}\"\n        (\"watermark\", \"pos\", \"tag\", \"estimatedBytes\", \"change\", \"precommit\",\n         \"writeTimeMs\")\n        VALUES (?, ?, 'commit', ?, ?, ?, ?)\n    `);\n    this.#cookies = new ChangeLogCookieWriter(db.db);\n  }\n\n  begin(watermark: string, json: string): void {\n    assertInvariant(\n      this.#watermark === undefined,\n      `change-log stream transaction already open at ${this.#watermark}`,\n    );\n    // This process is the sole writer of the change-log database, so the write\n    // lock is taken up front rather than risking a failed upgrade from a\n    // deferred transaction partway through the append.\n    this.#db.beginImmediate();\n    this.#watermark = watermark;\n    this.#pos = 0;\n    const change = extractChangeSubstring(json, 'begin');\n    const estimatedBytes = estimateChangeLogStreamRowBytes(\n      watermark,\n      'begin',\n      change,\n    );\n    this.#insertChange.run(\n      watermark,\n      this.#pos,\n      'begin',\n      estimatedBytes,\n      change,\n    );\n    this.#hasher = new ChangeLogTransactionHasher();\n    this.#hasher.add({\n      watermark,\n      pos: this.#pos,\n      tag: 'begin',\n      change,\n      precommit: null,\n    });\n    this.#estimatedBytes = estimatedBytes;\n  }\n\n  /**\n   * Appends one data or schema change. The parsed message is taken rather than\n   * just its tag because a schema change also mutates the cookie jar, and the\n   * caller already holds it — so the fold costs no re-parse.\n   *\n   * The cookie statements run here, i.e. inside this transaction and in stream\n   * order relative to the rows they were folded from. The Postgres change log\n   * has to flush its buffered batch to get the same ordering\n   * (`Storer.#processQueue()`); SQLite gets it from statement order.\n   */\n  append(json: string, change: DataOrSchemaChange): void {\n    const watermark = this.#requireWatermark();\n    const {tag} = change;\n    const stored = extractChangeSubstring(json, tag);\n    const estimatedBytes = estimateChangeLogStreamRowBytes(\n      watermark,\n      tag,\n      stored,\n    );\n    this.#insertChange.run(watermark, ++this.#pos, tag, estimatedBytes, stored);\n    this.#requireHasher().add({\n      watermark,\n      pos: this.#pos,\n      tag,\n      change: stored,\n      precommit: null,\n    });\n    this.#estimatedBytes += estimatedBytes;\n\n    if (isSchemaChange(change)) {\n      for (const op of this.#cookies.apply(change)) {\n        this.#cookieMutations.push(op.op);\n      }\n    }\n  }\n\n  commit(\n    watermark: string,\n    json: string,\n    writeTimeMs: number,\n  ): ChangeLogStreamTransactionStats {\n    const precommit = this.#requireWatermark();\n    assertInvariant(\n      watermark === precommit,\n      `change-log stream commit ${watermark} does not match begin ${precommit}`,\n    );\n    const change = extractChangeSubstring(json, 'commit');\n    const estimatedBytes = estimateChangeLogStreamRowBytes(\n      watermark,\n      'commit',\n      change,\n      precommit,\n      true,\n    );\n    this.#insertCommit.run(\n      watermark,\n      ++this.#pos,\n      estimatedBytes,\n      change,\n      precommit,\n      writeTimeMs,\n    );\n    const hasher = this.#requireHasher();\n    hasher.add({\n      watermark,\n      pos: this.#pos,\n      tag: 'commit',\n      change,\n      precommit,\n    });\n    const stats = {\n      rows: this.#pos + 1,\n      estimatedBytes: this.#estimatedBytes + estimatedBytes,\n      hash: hasher.digest(),\n      cookieMutations: this.#cookieMutations,\n    };\n    // The log is durable at `watermark` from here on, and nothing that can\n    // advance the watermark the stream would resume from has run yet. A crash in\n    // that window leaves a transaction the resumed stream re-delivers, which\n    // reconciliation truncates.\n    this.#db.commit();\n    this.#reset();\n    return stats;\n  }\n\n  /**\n   * Discards the open transaction, if any. Safe to call when none is open — the\n   * caller rolls back on an interrupted change stream, which can arrive between\n   * transactions as well as inside one.\n   */\n  rollback(): void {\n    const {inTransaction} = this.#db.db;\n    this.#reset();\n    if (inTransaction) {\n      this.#db.rollback();\n    }\n  }\n\n  #reset(): void {\n    this.#watermark = undefined;\n    this.#pos = 0;\n    this.#estimatedBytes = 0;\n    this.#hasher = undefined;\n    // A fresh array rather than a truncation: the committed stats hold a\n    // reference to the previous one.\n    this.#cookieMutations = [];\n  }\n\n  #requireWatermark(): string {\n    assertInvariant(\n      this.#watermark !== undefined,\n      'change-log stream message received outside of a transaction',\n    );\n    return this.#watermark;\n  }\n\n  #requireHasher(): ChangeLogTransactionHasher {\n    assertInvariant(\n      this.#hasher !== undefined,\n      'change-log stream hash received outside of a transaction',\n    );\n    return this.#hasher;\n  }\n}\n"],"mappings":";;;;;;AAyBA,IAAa,gCAAb,cAAmD,MAAM;CACvD,OAAyB;AAC3B;AAEA,SAAS,gBACP,WACA,SACmB;CACnB,IAAI,CAAC,WACH,MAAM,IAAI,8BAA8B,OAAO;AAEnD;;;;;;;;;;;;;;;;AAiBA,IAAa,wBAAb,MAAmC;CACjC;CACA;CACA;CACA;CAEA;CACA,OAAO;CACP,kBAAkB;CAClB;CACA,mBAAkC,CAAC;CAEnC,YAAY,IAAqB;EAC/B,KAAKA,MAAM;EACX,KAAKC,gBAAgB,GAAG,GAAG,QAAgB;qBAC1B,wBAAwB;;;KAGxC;EACD,KAAKC,gBAAgB,GAAG,GAAG,QAAgB;qBAC1B,wBAAwB;;;;KAIxC;EACD,KAAKC,WAAW,IAAI,sBAAsB,GAAG,EAAE;CACjD;CAEA,MAAM,WAAmB,MAAoB;EAC3C,gBACE,KAAKC,eAAe,KAAA,GACpB,iDAAiD,KAAKA,YACxD;EAIA,KAAKJ,IAAI,eAAe;EACxB,KAAKI,aAAa;EAClB,KAAKC,OAAO;EACZ,MAAM,SAAS,uBAAuB,MAAM,OAAO;EACnD,MAAM,iBAAiB,gCACrB,WACA,SACA,MACF;EACA,KAAKJ,cAAc,IACjB,WACA,KAAKI,MACL,SACA,gBACA,MACF;EACA,KAAKC,UAAU,IAAI,2BAA2B;EAC9C,KAAKA,QAAQ,IAAI;GACf;GACA,KAAK,KAAKD;GACV,KAAK;GACL;GACA,WAAW;EACb,CAAC;EACD,KAAKE,kBAAkB;CACzB;;;;;;;;;;;CAYA,OAAO,MAAc,QAAkC;EACrD,MAAM,YAAY,KAAKC,kBAAkB;EACzC,MAAM,EAAC,QAAO;EACd,MAAM,SAAS,uBAAuB,MAAM,GAAG;EAC/C,MAAM,iBAAiB,gCACrB,WACA,KACA,MACF;EACA,KAAKP,cAAc,IAAI,WAAW,EAAE,KAAKI,MAAM,KAAK,gBAAgB,MAAM;EAC1E,KAAKI,eAAe,EAAE,IAAI;GACxB;GACA,KAAK,KAAKJ;GACV;GACA,QAAQ;GACR,WAAW;EACb,CAAC;EACD,KAAKE,mBAAmB;EAExB,IAAI,eAAe,MAAM,GACvB,KAAK,MAAM,MAAM,KAAKJ,SAAS,MAAM,MAAM,GACzC,KAAKO,iBAAiB,KAAK,GAAG,EAAE;CAGtC;CAEA,OACE,WACA,MACA,aACiC;EACjC,MAAM,YAAY,KAAKF,kBAAkB;EACzC,gBACE,cAAc,WACd,4BAA4B,UAAU,wBAAwB,WAChE;EACA,MAAM,SAAS,uBAAuB,MAAM,QAAQ;EACpD,MAAM,iBAAiB,gCACrB,WACA,UACA,QACA,WACA,IACF;EACA,KAAKN,cAAc,IACjB,WACA,EAAE,KAAKG,MACP,gBACA,QACA,WACA,WACF;EACA,MAAM,SAAS,KAAKI,eAAe;EACnC,OAAO,IAAI;GACT;GACA,KAAK,KAAKJ;GACV,KAAK;GACL;GACA;EACF,CAAC;EACD,MAAM,QAAQ;GACZ,MAAM,KAAKA,OAAO;GAClB,gBAAgB,KAAKE,kBAAkB;GACvC,MAAM,OAAO,OAAO;GACpB,iBAAiB,KAAKG;EACxB;EAKA,KAAKV,IAAI,OAAO;EAChB,KAAKW,OAAO;EACZ,OAAO;CACT;;;;;;CAOA,WAAiB;EACf,MAAM,EAAC,kBAAiB,KAAKX,IAAI;EACjC,KAAKW,OAAO;EACZ,IAAI,eACF,KAAKX,IAAI,SAAS;CAEtB;CAEA,SAAe;EACb,KAAKI,aAAa,KAAA;EAClB,KAAKC,OAAO;EACZ,KAAKE,kBAAkB;EACvB,KAAKD,UAAU,KAAA;EAGf,KAAKI,mBAAmB,CAAC;CAC3B;CAEA,oBAA4B;EAC1B,gBACE,KAAKN,eAAe,KAAA,GACpB,6DACF;EACA,OAAO,KAAKA;CACd;CAEA,iBAA6C;EAC3C,gBACE,KAAKE,YAAY,KAAA,GACjB,0DACF;EACA,OAAO,KAAKA;CACd;AACF"}

package/out/zero-cache/src/services/replicator/change-processor.d.ts

@@ -9,6 +9,12 @@ export type CommitResult = {
     completedBackfill: DownloadStatus | undefined;
     schemaUpdated: boolean;
     changeLogUpdated: boolean;
+    /**
+     * Millisecond epoch at which the transaction committed upstream, if the
+     * ChangeSource reported one. Propagated to ViewSyncers as the origin
+     * timestamp of the end-to-end serving lag measurement.
+     */
+    upstreamCommitTimeMs?: number | undefined;
 };
 /**
  * The ChangeProcessor partitions the stream of messages into transactions

package/out/zero-cache/src/services/replicator/change-processor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-processor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/change-processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uCAAuC,CAAC;AAiB1E,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AAiC5D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,iBAAiB,CAAC;AASpD,MAAM,MAAM,mBAAmB,GAAG,cAAc,GAAG,cAAc,CAAC;AAElE,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,cAAc,GAAG,SAAS,CAAC;IAC9C,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,eAAe;;gBAiBxB,EAAE,EAAE,eAAe,EACnB,IAAI,EAAE,mBAAmB,EACzB,WAAW,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,KAAK,IAAI;IAgCrD,KAAK,CAAC,EAAE,EAAE,UAAU;IAIpB,8CAA8C;IAC9C,cAAc,CACZ,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,gBAAgB,GAC3B,YAAY,GAAG,IAAI;CAyJvB"}
+{"version":3,"file":"change-processor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/change-processor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uCAAuC,CAAC;AAoB1E,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AAiC5D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAUpD,MAAM,MAAM,mBAAmB,GAAG,cAAc,GAAG,cAAc,CAAC;AASlE,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,cAAc,GAAG,SAAS,CAAC;IAC9C,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3C,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,eAAe;;gBAkBxB,EAAE,EAAE,eAAe,EACnB,IAAI,EAAE,mBAAmB,EACzB,WAAW,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,KAAK,IAAI;IAmCrD,KAAK,CAAC,EAAE,EAAE,UAAU;IAIpB,8CAA8C;IAC9C,cAAc,CACZ,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,gBAAgB,GAC3B,YAAY,GAAG,IAAI;CA2JvB"}

package/out/zero-cache/src/services/replicator/change-processor.js

@@ -6,16 +6,20 @@ import { stringify } from "../../../../shared/src/bigint-json.js";
 import { ZERO_VERSION_COLUMN_NAME } from "./schema/constants.js";
 import { liteRow } from "../../types/lite.js";
 import { liteTableName } from "../../types/names.js";
-import { mapPostgresToLite, mapPostgresToLiteColumn, mapPostgresToLiteIndex } from "../../db/pg-to-lite.js";
+import { isArrayColumn, isEnumColumn, mapPostgresToLite, mapPostgresToLiteColumn, mapPostgresToLiteIndex } from "../../db/pg-to-lite.js";
 import { ColumnMetadataStore } from "./schema/column-metadata.js";
 import { TableMetadataTracker } from "./schema/table-metadata.js";
 import { computeZqlSpecs, listIndexes, listTables } from "../../db/lite-tables.js";
+import { BackfillingTracker } from "./schema/backfilling.js";
 import { ChangeLog } from "./schema/change-log.js";
 import { updateReplicationWatermark } from "./schema/replication-state.js";
 import { id } from "../../types/sql.js";
 import { createLiteIndexStatement, createLiteTableStatement, liteColumnDef } from "../../db/create.js";
 import { SqliteError } from "@rocicorp/zero-sqlite3";
+import { constants } from "node:buffer";
 //#region ../zero-cache/src/services/replicator/change-processor.ts
+var BIND_VALUE_TOO_BIG_ERROR = "The bound string, buffer, or bigint is too big";
+var MAX_SQLITE_BIND_BYTES = Math.min(constants.MAX_LENGTH, constants.MAX_STRING_LENGTH);
 /**
 * The ChangeProcessor partitions the stream of messages into transactions
 * by creating a {@link TransactionProcessor} when a transaction begins, and dispatching
@@ -31,6 +35,7 @@ var ChangeProcessor = class {
 	#db;
 	#changeLog;
 	#tableMetadata;
+	#backfilling;
 	#mode;
 	#failService;
 	#tableSpecs = /* @__PURE__ */ new Map();
@@ -40,16 +45,16 @@ var ChangeProcessor = class {
 		this.#db = db;
 		this.#changeLog = new ChangeLog(db.db);
 		this.#tableMetadata = new TableMetadataTracker(db.db);
+		this.#backfilling = new BackfillingTracker(db.db);
 		this.#mode = mode;
 		this.#failService = failService;
 	}
 	#fail(lc, err) {
 		if (!this.#failure) {
 			let failureError = err;
-			try {
-				this.#currentTx?.abort(lc);
-			} catch (rollbackError) {
-				const combinedError = /* @__PURE__ */ new Error(`Message processing failed and rollback also failed: operation error = ${String(err)}; rollback error = ${String(rollbackError)}`);
+			const rollbackErrors = [attempt(() => this.#currentTx?.abort(lc))].filter((e) => e !== void 0);
+			if (rollbackErrors.length) {
+				const combinedError = /* @__PURE__ */ new Error(`Message processing failed and rollback also failed: operation error = ${String(err)}; rollback error = ${rollbackErrors.map(String).join("; ")}`);
 				combinedError.cause = err;
 				failureError = combinedError;
 			}
@@ -81,7 +86,7 @@ var ChangeProcessor = class {
 	#beginTransaction(lc, commitVersion, jsonFormat) {
 		const start = Date.now();
 		for (let i = 0;; i++) try {
-			return new TransactionProcessor(lc, this.#db, this.#mode, this.#changeLog, this.#tableMetadata, this.#tableSpecs, commitVersion, jsonFormat);
+			return new TransactionProcessor(lc, this.#db, this.#mode, this.#changeLog, this.#tableMetadata, this.#backfilling, this.#tableSpecs, commitVersion, jsonFormat);
 		} catch (e) {
 			if (e instanceof SqliteError && e.code === "SQLITE_BUSY") {
 				lc.warn?.(`SQLITE_BUSY for ${Date.now() - start} ms (attempt ${i + 1}). This is only expected if litestream is performing a large checkpoint.`, e);
@@ -100,12 +105,13 @@ var ChangeProcessor = class {
 		const tx = this.#currentTx;
 		if (!tx) throw new Error(`Received message outside of transaction: ${stringify(msg)}`);
 		if (msg.tag === "commit") {
-			this.#currentTx = null;
 			assert(watermark, "watermark is required for commit messages");
-			return tx.processCommit(msg, watermark);
+			const result = tx.processCommit(msg, watermark);
+			this.#currentTx = null;
+			return result;
 		}
 		if (msg.tag === "rollback") {
-			this.#currentTx?.abort(lc);
+			tx.abort(lc);
 			this.#currentTx = null;
 			return null;
 		}
@@ -189,19 +195,20 @@ var TransactionProcessor = class {
 	#version;
 	#changeLog;
 	#tableMetadata;
+	#backfilling;
 	#tableSpecs;
 	#jsonFormat;
 	#columnMetadata;
 	#pos = 0;
 	#schemaChanged = false;
 	#numChangeLogEntries = 0;
-	constructor(lc, db, mode, changeLog, tableMetadata, tableSpecs, commitVersion, jsonFormat) {
+	constructor(lc, db, mode, changeLog, tableMetadata, backfilling, tableSpecs, commitVersion, jsonFormat) {
 		this.#startMs = Date.now();
 		this.#mode = mode;
 		this.#jsonFormat = jsonFormat;
 		switch (mode) {
 			case "serving":
-				db.beginConcurrent();
+				db.beginImmediate();
 				break;
 			case "backup":
 				db.beginImmediate();
@@ -214,6 +221,7 @@ var TransactionProcessor = class {
 		this.#lc = lc.withContext("version", commitVersion);
 		this.#changeLog = changeLog;
 		this.#tableMetadata = tableMetadata;
+		this.#backfilling = backfilling;
 		this.#tableSpecs = tableSpecs;
 		this.#columnMetadata = must(ColumnMetadataStore.getInstance(db.db));
 		if (this.#tableSpecs.size === 0) this.#reloadTableSpecs();
@@ -274,12 +282,21 @@ var TransactionProcessor = class {
 		const currKey = oldKey ?? newKey;
 		const conds = Object.keys(currKey).map((col) => `${id(col)}=?`);
 		const setExprs = Object.keys(row).map((col) => `${id(col)}=?`);
-		const { changes } = this.#db.run(`
-      UPDATE ${id(table)}
-        SET ${setExprs.join(",")}
-        WHERE ${conds.join(" AND ")}
-      `, [...Object.values(row), ...Object.values(currKey)]);
-		if (changes === 0) this.#upsert(table, row);
+		try {
+			const { changes } = this.#db.run(`
+        UPDATE ${id(table)}
+          SET ${setExprs.join(",")}
+          WHERE ${conds.join(" AND ")}
+        `, [...Object.values(row), ...Object.values(currKey)]);
+			if (changes === 0) this.#upsert(table, row);
+		} catch (e) {
+			if (!(e instanceof RangeError) || e.message !== BIND_VALUE_TOO_BIG_ERROR) throw e;
+			const binding = [...Object.entries(row), ...Object.entries(currKey)].map(([column, value]) => [column, oversizedValueDescription(value)]).find(([, description]) => description);
+			const relationOid = "relationOid" in update.relation && typeof update.relation.relationOid === "number" ? ` relationOid=${update.relation.relationOid}` : "";
+			const error = new Error(`Oversized SQLite update binding: tx=${this.#version}${relationOid} table=${update.relation.schema}.${update.relation.name} column=${binding?.[0] ?? "unknown"}${binding?.[1] ? ` ${binding[1]}` : ""}`, { cause: e });
+			error.name = "OversizedUpdateBindingError";
+			throw error;
+		}
 	}
 	processDelete(del) {
 		const table = liteTableName(del.relation);
@@ -301,6 +318,7 @@ var TransactionProcessor = class {
 	}
 	processCreateTable(create) {
 		if (create.metadata) this.#tableMetadata.setUpstreamMetadata(create.spec, create.metadata);
+		this.#backfilling.apply(create);
 		const table = mapPostgresToLite(create.spec);
 		this.#db.db.exec(createLiteTableStatement(table));
 		for (const [colName, colSpec] of Object.entries(create.spec.columns)) this.#columnMetadata.insert(table.name, colName, colSpec, create.backfill?.[colName]);
@@ -310,9 +328,11 @@ var TransactionProcessor = class {
 	}
 	processTableMetadata(msg) {
 		this.#tableMetadata.setUpstreamMetadata(msg.table, msg.new);
+		this.#backfilling.apply(msg);
 	}
 	processRenameTable(rename) {
 		this.#tableMetadata.rename(rename.old, rename.new);
+		this.#backfilling.apply(rename);
 		const oldName = liteTableName(rename.old);
 		const newName = liteTableName(rename.new);
 		this.#db.db.exec(`ALTER TABLE ${id(oldName)} RENAME TO ${id(newName)}`);
@@ -323,6 +343,7 @@ var TransactionProcessor = class {
 	}
 	processAddColumn(msg) {
 		if (msg.tableMetadata) this.#tableMetadata.setUpstreamMetadata(msg.table, msg.tableMetadata);
+		this.#backfilling.apply(msg);
 		const table = liteTableName(msg.table);
 		const { name } = msg.column;
 		const spec = mapPostgresToLiteColumn(table, msg.column);
@@ -333,22 +354,24 @@ var TransactionProcessor = class {
 		this.#lc.info?.(msg.tag, table, msg.column);
 	}
 	processUpdateColumn(msg) {
+		this.#backfilling.apply(msg);
 		const table = liteTableName(msg.table);
 		let oldName = msg.old.name;
 		const newName = msg.new.name;
-		const oldSpec = mapPostgresToLiteColumn(table, msg.old, "ignore-default");
-		const newSpec = mapPostgresToLiteColumn(table, msg.new, "ignore-default");
-		if (oldName === newName && oldSpec.dataType === newSpec.dataType) {
+		const storageTypesDiffer = differentStorageTypes(msg.old.spec, msg.new.spec);
+		if (oldName === newName && !storageTypesDiffer) {
 			this.#columnMetadata.update(table, msg.old.name, msg.new.name, msg.new.spec);
-			this.#lc.info?.(msg.tag, "updated metadata only", oldSpec, newSpec);
+			if (Boolean(msg.old.spec.notNull) !== Boolean(msg.new.spec.notNull)) this.#bumpVersions(msg.table);
+			this.#lc.info?.(msg.tag, "updated metadata only", msg.old, msg.new);
 			return;
 		}
-		if (oldSpec.dataType !== newSpec.dataType) {
+		if (storageTypesDiffer) {
+			const newLiteSpec = mapPostgresToLiteColumn(table, msg.new, "ignore-default");
 			const tableSpec = must(listTables(this.#db.db, false, false).find((tableSpec) => tableSpec.name === table));
 			const indexes = listIndexes(this.#db.db).filter((idx) => idx.tableName === table);
 			const tmpTable = `tmp.${table}`;
 			const newColumns = mapEntries(tableSpec.columns, (column, spec) => [column === oldName ? newName : column, column === oldName ? {
-				...newSpec,
+				...newLiteSpec,
 				pos: spec.pos
 			} : spec]);
 			const sourceColumns = Object.keys(tableSpec.columns);
@@ -381,11 +404,13 @@ var TransactionProcessor = class {
 		const { column } = msg;
 		this.#db.db.exec(`ALTER TABLE ${id(table)} DROP ${id(column)}`);
 		this.#columnMetadata.deleteColumn(table, column);
+		this.#backfilling.apply(msg);
 		this.#bumpVersions(msg.table);
 		this.#lc.info?.(msg.tag, table, column);
 	}
 	processDropTable(drop) {
 		this.#tableMetadata.drop(drop.id);
+		this.#backfilling.apply(drop);
 		const name = liteTableName(drop.id);
 		this.#db.db.exec(`DROP TABLE IF EXISTS ${id(name)}`);
 		this.#columnMetadata.deleteTable(name);
@@ -472,11 +497,13 @@ var TransactionProcessor = class {
 		this.#lc.debug?.(`backfilled ${backfilled} rows (skipped ${skipped}) into ${tableName}`);
 	}
 	#completedBackfill;
-	processBackfillCompleted({ relation, columns, status }) {
+	processBackfillCompleted(msg) {
+		const { relation, columns, status } = msg;
 		const tableName = liteTableName(relation);
 		const cols = [...relation.rowKey.columns, ...columns];
 		const columnMetadata = must(ColumnMetadataStore.getInstance(this.#db.db));
 		for (const col of cols) columnMetadata.clearBackfilling(tableName, col);
+		this.#backfilling.apply(msg);
 		this.#bumpVersions(relation);
 		if (status) this.#completedBackfill = {
 			table: tableName,
@@ -500,7 +527,8 @@ var TransactionProcessor = class {
 			watermark,
 			completedBackfill: this.#completedBackfill,
 			schemaUpdated: this.#schemaChanged,
-			changeLogUpdated: this.#numChangeLogEntries > 0
+			changeLogUpdated: this.#numChangeLogEntries > 0,
+			upstreamCommitTimeMs: commit.commitTimeMs
 		};
 	}
 	abort(lc) {
@@ -508,10 +536,27 @@ var TransactionProcessor = class {
 		this.#db.rollback();
 	}
 };
+function differentStorageTypes(oldSpec, newSpec) {
+	return oldSpec.dataType !== newSpec.dataType || isEnumColumn(oldSpec) !== isEnumColumn(newSpec) || isArrayColumn(oldSpec) !== isArrayColumn(newSpec);
+}
 function getBackfilledColumns(row, { backfilling }) {
 	if (!backfilling?.length) return;
 	return backfilling.filter((col) => col in row);
 }
+function oversizedValueDescription(value) {
+	if (typeof value === "bigint") return BigInt.asIntN(64, value) !== value ? "valueType=bigint fitsInt64=false" : void 0;
+	const sizeBytes = typeof value === "string" ? Buffer.byteLength(value) : value instanceof Uint8Array ? value.byteLength : void 0;
+	if (sizeBytes !== void 0 && sizeBytes > MAX_SQLITE_BIND_BYTES) return `valueType=${typeof value === "string" ? "string" : "buffer"} sizeBytes=${sizeBytes} limitBytes=${MAX_SQLITE_BIND_BYTES}`;
+}
+/** Runs `fn`, returning its error rather than throwing it. */
+function attempt(fn) {
+	try {
+		fn();
+		return;
+	} catch (e) {
+		return e ?? /* @__PURE__ */ new Error("rollback failed");
+	}
+}
 function ensureError(err) {
 	if (err instanceof Error) return err;
 	const error = /* @__PURE__ */ new Error();