@rocicorp/zero 1.11.0-canary.3 → 1.11.0-canary.31

package/out/zero-cache/src/services/change-source/protocol/backfill-progress.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"backfill-progress.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/protocol/backfill-progress.ts"],"sourcesContent":["import {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {\n  BackfillProgressMark,\n  BackfillRequest,\n  Identifier,\n} from './current.ts';\n\n/**\n * Whether two progress marks are on the same timeline, and are thus comparable.\n */\nfunction sameTimeline(a: BackfillProgressMark, b: BackfillProgressMark) {\n  return a.timeline === b.timeline;\n}\n\n/** Returns the earlier of two marks on the same timeline. */\nfunction earlier(\n  a: BackfillProgressMark,\n  b: BackfillProgressMark,\n): BackfillProgressMark {\n  return b.progressMark < a.progressMark ? b : a;\n}\n\n/** Returns the later of two marks on the same timeline. */\nfunction later(\n  a: BackfillProgressMark,\n  b: BackfillProgressMark,\n): BackfillProgressMark {\n  return b.progressMark > a.progressMark ? b : a;\n}\n\n/**\n * The result of applying a backfill message to a column's tracked progress.\n * If `accept` is true, `progress` is the column's new progress.\n */\nexport type AcceptResult =\n  | {accept: true; progress: BackfillProgressMark | undefined}\n  | {accept: false};\n\n/**\n * Applies the continuity policy of `backfill` (and `backfill-completed`)\n * messages to a column's `own` tracked progress, where `undefined` means that\n * the column has not received any backfill data (i.e. it needs a backfill\n * from scratch).\n *\n * * Messages without `progressMarks` are from pre-v8 replication-managers\n *   and are accepted unconditionally (i.e. the legacy behavior), with no\n *   change to the tracked progress.\n * * A message without a `previous` mark starts a backfill from scratch (or on\n *   a new timeline), and is always accepted.\n * * Otherwise, the message is only accepted if it is on the same timeline as\n *   `own`, and continues from a position at or before `own` (i.e. it does not\n *   create a gap).\n *\n * `current` is unset for `backfill-completed` messages, which do not advance\n * the progress.\n */\nexport function acceptBackfill(\n  own: BackfillProgressMark | undefined,\n  marks:\n    | {\n        previous?: BackfillProgressMark | undefined;\n        current?: BackfillProgressMark;\n      }\n    | undefined,\n): AcceptResult {\n  if (marks === undefined) {\n    return {accept: true, progress: own}; // legacy (pre-v8)\n  }\n  const {previous, current} = marks;\n  // Starting from scratch is always accepted, as there can be no gap.\n  if (previous === undefined) {\n    return {accept: true, progress: current};\n  }\n  if (\n    own === undefined ||\n    !sameTimeline(own, previous) ||\n    previous.progressMark > own.progressMark\n  ) {\n    return {accept: false};\n  }\n  return {\n    accept: true,\n    progress:\n      current === undefined\n        ? own\n        : sameTimeline(own, current)\n          ? later(own, current)\n          : current,\n  };\n}\n\n/**\n * Returns whether a stream whose progress (for a column) is at `stream`\n * will deliver all of the data needed by a subscriber whose progress (for the\n * same column) is at `subscriber`. A `stream` progress of `undefined` indicates\n * a backfill that will start from scratch, which covers all subscribers.\n */\nexport function covers(\n  stream: BackfillProgressMark | undefined,\n  subscriber: BackfillProgressMark | undefined,\n): boolean {\n  return (\n    stream === undefined ||\n    (subscriber !== undefined &&\n      sameTimeline(stream, subscriber) &&\n      stream.progressMark <= subscriber.progressMark)\n  );\n}\n\n/**\n * Returns the position from which a backfill for the `req` must start\n * in order to satisfy the progress of all of its columns, i.e. the earliest\n * progress mark, or `undefined` (i.e. from scratch) if any column has no\n * progress or if the columns are on different timelines.\n */\nexport function resumePoint(\n  req: BackfillRequest,\n): BackfillProgressMark | undefined {\n  let point: BackfillProgressMark | undefined;\n  for (const {progress} of Object.values(req.columns)) {\n    if (progress === undefined) {\n      return undefined;\n    }\n    if (point === undefined) {\n      point = progress;\n    } else if (!sameTimeline(point, progress)) {\n      return undefined;\n    } else {\n      point = earlier(point, progress);\n    }\n  }\n  return point;\n}\n\n/**\n * Returns a copy of the `req` with the progress of every column set to\n * the request's {@link resumePoint}, i.e. the position from which the\n * backfill of the request will actually start.\n */\nexport function withResumePoint(req: BackfillRequest): BackfillRequest {\n  const progress = resumePoint(req);\n  return {\n    table: req.table,\n    columns: Object.fromEntries(\n      Object.entries(req.columns).map(([col, {id}]) => [\n        col,\n        progress === undefined ? {id} : {id, progress},\n      ]),\n    ),\n  };\n}\n\n/**\n * Merges two BackfillRequests for the same table into one that satisfies\n * both. Both requests are expected to reflect the same (current) upstream\n * schema. For columns in both requests, the earlier progress mark is chosen,\n * or no progress mark (i.e. from scratch) if they are on different timelines.\n * If a column's backfill IDs differ (which is unexpected), the ID of `req1`\n * is chosen, and the column is backfilled from scratch.\n */\nexport function getTableSuperset(\n  req1: BackfillRequest,\n  req2: BackfillRequest,\n): BackfillRequest {\n  const columns = {...req1.columns};\n  for (const [name, col] of Object.entries(req2.columns)) {\n    const exists = columns[name];\n    if (!exists) {\n      columns[name] = col;\n      continue;\n    }\n    const {id, progress} = exists;\n    if (\n      progress === undefined ||\n      col.progress === undefined ||\n      !sameTimeline(progress, col.progress) ||\n      !deepEqual(id, col.id)\n    ) {\n      // Different timelines (or IDs). Start from scratch.\n      columns[name] = {id};\n    } else {\n      columns[name] = {id, progress: earlier(progress, col.progress)};\n    }\n  }\n  return {\n    table: {\n      ...req1.table,\n      metadata: req1.table.metadata ?? req2.table.metadata,\n    },\n    columns,\n  };\n}\n\nfunction tableKey({schema, name}: Identifier) {\n  return JSON.stringify([schema, name]);\n}\n\n/**\n * Merges the BackfillRequests of multiple parties (e.g. subscribers) into\n * one request per table, using {@link getTableSuperset}.\n */\nexport function getSuperset(\n  ...requestSets: Iterable<BackfillRequest>[]\n): BackfillRequest[] {\n  const merged = new Map<string, BackfillRequest>();\n  for (const requests of requestSets) {\n    for (const req of requests) {\n      const key = tableKey(req.table);\n      const existing = merged.get(key);\n      merged.set(key, existing ? getTableSuperset(existing, req) : req);\n    }\n  }\n  return [...merged.values()];\n}\n"],"mappings":";;;;;AAUA,SAAS,aAAa,GAAyB,GAAyB;CACtE,OAAO,EAAE,aAAa,EAAE;AAC1B;;AAGA,SAAS,QACP,GACA,GACsB;CACtB,OAAO,EAAE,eAAe,EAAE,eAAe,IAAI;AAC/C;;AAGA,SAAS,MACP,GACA,GACsB;CACtB,OAAO,EAAE,eAAe,EAAE,eAAe,IAAI;AAC/C;;;;;;;;;;;;;;;;;;;AA4BA,SAAgB,eACd,KACA,OAMc;CACd,IAAI,UAAU,KAAA,GACZ,OAAO;EAAC,QAAQ;EAAM,UAAU;CAAG;CAErC,MAAM,EAAC,UAAU,YAAW;CAE5B,IAAI,aAAa,KAAA,GACf,OAAO;EAAC,QAAQ;EAAM,UAAU;CAAO;CAEzC,IACE,QAAQ,KAAA,KACR,CAAC,aAAa,KAAK,QAAQ,KAC3B,SAAS,eAAe,IAAI,cAE5B,OAAO,EAAC,QAAQ,MAAK;CAEvB,OAAO;EACL,QAAQ;EACR,UACE,YAAY,KAAA,IACR,MACA,aAAa,KAAK,OAAO,IACvB,MAAM,KAAK,OAAO,IAClB;CACV;AACF;;;;;;;AAQA,SAAgB,OACd,QACA,YACS;CACT,OACE,WAAW,KAAA,KACV,eAAe,KAAA,KACd,aAAa,QAAQ,UAAU,KAC/B,OAAO,gBAAgB,WAAW;AAExC;;;;;;;AAQA,SAAgB,YACd,KACkC;CAClC,IAAI;CACJ,KAAK,MAAM,EAAC,cAAa,OAAO,OAAO,IAAI,OAAO,GAAG;EACnD,IAAI,aAAa,KAAA,GACf;EAEF,IAAI,UAAU,KAAA,GACZ,QAAQ;OACH,IAAI,CAAC,aAAa,OAAO,QAAQ,GACtC;OAEA,QAAQ,QAAQ,OAAO,QAAQ;CAEnC;CACA,OAAO;AACT;;;;;;AAOA,SAAgB,gBAAgB,KAAuC;CACrE,MAAM,WAAW,YAAY,GAAG;CAChC,OAAO;EACL,OAAO,IAAI;EACX,SAAS,OAAO,YACd,OAAO,QAAQ,IAAI,OAAO,EAAE,KAAK,CAAC,KAAK,EAAC,UAAS,CAC/C,KACA,aAAa,KAAA,IAAY,EAAC,GAAE,IAAI;GAAC;GAAI;EAAQ,CAC/C,CAAC,CACH;CACF;AACF;;;;;;;;;AAUA,SAAgB,iBACd,MACA,MACiB;CACjB,MAAM,UAAU,EAAC,GAAG,KAAK,QAAO;CAChC,KAAK,MAAM,CAAC,MAAM,QAAQ,OAAO,QAAQ,KAAK,OAAO,GAAG;EACtD,MAAM,SAAS,QAAQ;EACvB,IAAI,CAAC,QAAQ;GACX,QAAQ,QAAQ;GAChB;EACF;EACA,MAAM,EAAC,IAAI,aAAY;EACvB,IACE,aAAa,KAAA,KACb,IAAI,aAAa,KAAA,KACjB,CAAC,aAAa,UAAU,IAAI,QAAQ,KACpC,CAAC,UAAU,IAAI,IAAI,EAAE,GAGrB,QAAQ,QAAQ,EAAC,GAAE;OAEnB,QAAQ,QAAQ;GAAC;GAAI,UAAU,QAAQ,UAAU,IAAI,QAAQ;EAAC;CAElE;CACA,OAAO;EACL,OAAO;GACL,GAAG,KAAK;GACR,UAAU,KAAK,MAAM,YAAY,KAAK,MAAM;EAC9C;EACA;CACF;AACF;AAEA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,KAAK,UAAU,CAAC,QAAQ,IAAI,CAAC;AACtC;;;;;AAMA,SAAgB,YACd,GAAG,aACgB;CACnB,MAAM,yBAAS,IAAI,IAA6B;CAChD,KAAK,MAAM,YAAY,aACrB,KAAK,MAAM,OAAO,UAAU;EAC1B,MAAM,MAAM,SAAS,IAAI,KAAK;EAC9B,MAAM,WAAW,OAAO,IAAI,GAAG;EAC/B,OAAO,IAAI,KAAK,WAAW,iBAAiB,UAAU,GAAG,IAAI,GAAG;CAClE;CAEF,OAAO,CAAC,GAAG,OAAO,OAAO,CAAC;AAC5B"}

package/out/zero-cache/src/services/change-source/protocol/current/data.d.ts

@@ -67,7 +67,7 @@ export declare const updateSchema: v.ObjectType<{
             type?: "default" | "full" | "index" | "nothing" | undefined;
         };
     }>;
-    key: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue> | null>;
+    key: v.UnionType<[v.Type<null>, v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>]>;
     new: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>;
 }, undefined>;
 export declare const deleteSchema: v.ObjectType<{
@@ -104,6 +104,11 @@ export declare const identifierSchema: v.ObjectType<{
 export type Identifier = v.Infer<typeof identifierSchema>;
 export declare const backfillIDSchema: v.Type<Record<string, import("./json.ts").JSONValue | undefined>>;
 export type BackfillID = v.Infer<typeof backfillIDSchema>;
+export declare const backfillProgressMarkSchema: v.ObjectType<{
+    progressMark: v.Type<string>;
+    timeline: v.Optional<string>;
+}, v.UnionType<[v.Type<number>, v.Type<string>]>>;
+export type BackfillProgressMark = v.Infer<typeof backfillProgressMarkSchema>;
 export declare const createTableSchema: v.ObjectType<{
     tag: v.Type<"create-table">;
     spec: v.ObjectType<Omit<{
@@ -259,6 +264,18 @@ export declare const backfillSchema: v.ObjectType<{
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
     rowValues: v.ArrayType<v.ArrayType<v.Type<import("../../../../../../shared/src/bigint-json.ts").JSONValue>>>;
+    progressMarks: v.Optional<{
+        previous?: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        } | undefined;
+        current: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        };
+    }>;
     status: v.Optional<{
         rows: number;
         totalRows: number;
@@ -277,6 +294,13 @@ export declare const backfillCompletedSchema: v.ObjectType<{
     }, undefined>;
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
+    progressMarks: v.Optional<{
+        previous?: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        } | undefined;
+    }>;
     status: v.Optional<{
         rows: number;
         totalRows: number;
@@ -327,7 +351,7 @@ export declare const dataChangeSchema: v.UnionType<[v.ObjectType<{
             type?: "default" | "full" | "index" | "nothing" | undefined;
         };
     }>;
-    key: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue> | null>;
+    key: v.UnionType<[v.Type<null>, v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>]>;
     new: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>;
 }, undefined>, v.ObjectType<{
     tag: v.Type<"delete">;
@@ -367,6 +391,18 @@ export declare const dataChangeSchema: v.UnionType<[v.ObjectType<{
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
     rowValues: v.ArrayType<v.ArrayType<v.Type<import("../../../../../../shared/src/bigint-json.ts").JSONValue>>>;
+    progressMarks: v.Optional<{
+        previous?: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        } | undefined;
+        current: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        };
+    }>;
     status: v.Optional<{
         rows: number;
         totalRows: number;
@@ -516,6 +552,13 @@ export declare const schemaChangeSchema: v.UnionType<[v.ObjectType<{
     }, undefined>;
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
+    progressMarks: v.Optional<{
+        previous?: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        } | undefined;
+    }>;
     status: v.Optional<{
         rows: number;
         totalRows: number;

package/out/zero-cache/src/services/change-source/protocol/current/data.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/data.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,6CAA6C,CAAC;AAErD,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAE5D,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,gCAAgC,CAAC;AAI9D,eAAO,MAAM,WAAW;;;;aAgBtB,CAAC;AAEH,eAAO,MAAM,YAAY;;;aAevB,CAAC;AAEH,eAAO,MAAM,cAAc;;aAEzB,CAAC;AAcH,eAAO,MAAM,cAAc;;;;;;;;;EA2BvB,CAAC;AAGL,eAAO,MAAM,iBAAiB;;;;;;;aAK5B,CAAC;AAUH,eAAO,MAAM,mBAAmB;;2EAER,CAAC;AAEzB,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,eAAO,MAAM,SAAS,yFAA4B,CAAC;AAEnD,eAAO,MAAM,YAAY;;;;;;;;;;;;;aAIvB,CAAC;AAEH,eAAO,MAAM,YAAY;;;;;;;;;;;;;;aAUvB,CAAC;AAEH,eAAO,MAAM,YAAY;;;;;;;;;;;;;aAKvB,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;;aAGzB,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAU1D,eAAO,MAAM,gBAAgB,mEAAmB,CAAC;AAEjD,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;aA4B5B,CAAC;AAEH,eAAO,MAAM,iBAAiB;;;;;;;;;;aAI5B,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;aAKpC,CAAC;AAOH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;aAa1B,CAAC;AAEH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;aAK7B,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;;;;;aAI3B,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;aAG1B,CAAC;AAEH,eAAO,MAAM,iBAAiB;;;;;;;;;;;aAG5B,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;aAG1B,CAAC;AAEH,eAAO,MAAM,oBAAoB;;;;aAI/B,CAAC;AAEH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAIlE,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;aA8BzB,CAAC;AAIH,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;aAkBlC,CAAC;AAEH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC7D,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAM5B,CAAC;AAWF,QAAA,MAAM,oBAAoB,kEAAoC,CAAC;AAE/D,MAAM,MAAM,UAAU,GAAG,SAAS,CAChC,UAAU,EAAE,+CAA+C;AAC3D,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CACjC,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAejE,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAA4B,CAAC;AAE5D,QAAA,MAAM,sBAAsB,0LAAsC,CAAC;AAEnE,MAAM,MAAM,YAAY,GAAG,SAAS,CAClC,UAAU,EACV,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CACnC,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAErE,MAAM,MAAM,kBAAkB,GAAG,UAAU,GAAG,YAAY,CAAC;AAE3D,MAAM,MAAM,MAAM,GACd,YAAY,GACZ,kBAAkB,GAClB,aAAa,GACb,eAAe,CAAC;AAEpB,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;AAItC,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,YAAY,CAErE;AAID,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,UAAU,CAEjE"}
+{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/data.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,6CAA6C,CAAC;AAErD,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAE5D,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,gCAAgC,CAAC;AAI9D,eAAO,MAAM,WAAW;;;;aAgBtB,CAAC;AAEH,eAAO,MAAM,YAAY;;;aAevB,CAAC;AAEH,eAAO,MAAM,cAAc;;aAEzB,CAAC;AAcH,eAAO,MAAM,cAAc;;;;;;;;;EA2BvB,CAAC;AAGL,eAAO,MAAM,iBAAiB;;;;;;;aAK5B,CAAC;AAUH,eAAO,MAAM,mBAAmB;;2EAER,CAAC;AAEzB,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,eAAO,MAAM,SAAS,yFAA4B,CAAC;AAEnD,eAAO,MAAM,YAAY;;;;;;;;;;;;;aAIvB,CAAC;AAEH,eAAO,MAAM,YAAY;;;;;;;;;;;;;;aAUvB,CAAC;AAEH,eAAO,MAAM,YAAY;;;;;;;;;;;;;aAKvB,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;;aAGzB,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;aAG3B,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAU1D,eAAO,MAAM,gBAAgB,mEAAmB,CAAC;AAEjD,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,eAAO,MAAM,0BAA0B;;;iDAiBC,CAAC;AAEzC,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAE9E,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;aA4B5B,CAAC;AAEH,eAAO,MAAM,iBAAiB;;;;;;;;;;aAI5B,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;aAKpC,CAAC;AAOH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;aAa1B,CAAC;AAEH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;aAK7B,CAAC;AAEH,eAAO,MAAM,gBAAgB;;;;;;;aAI3B,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;aAG1B,CAAC;AAEH,eAAO,MAAM,iBAAiB;;;;;;;;;;;aAG5B,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;aAG1B,CAAC;AAEH,eAAO,MAAM,oBAAoB;;;;aAI/B,CAAC;AAEH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAIlE,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;aAwEzB,CAAC;AAIH,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;aAmClC,CAAC;AAEH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC7D,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AACzD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAE7D,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAC1D,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAM5B,CAAC;AAWF,QAAA,MAAM,oBAAoB,kEAAoC,CAAC;AAE/D,MAAM,MAAM,UAAU,GAAG,SAAS,CAChC,UAAU,EAAE,+CAA+C;AAC3D,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CACjC,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAejE,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAA4B,CAAC;AAE5D,QAAA,MAAM,sBAAsB,0LAAsC,CAAC;AAEnE,MAAM,MAAM,YAAY,GAAG,SAAS,CAClC,UAAU,EACV,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CACnC,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAErE,MAAM,MAAM,kBAAkB,GAAG,UAAU,GAAG,YAAY,CAAC;AAE3D,MAAM,MAAM,MAAM,GACd,YAAY,GACZ,kBAAkB,GAClB,aAAa,GACb,eAAe,CAAC;AAEpB,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;AAItC,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,YAAY,CAErE;AAID,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,UAAU,CAEjE"}

package/out/zero-cache/src/services/change-source/protocol/current/data.js

@@ -78,6 +78,10 @@ var identifierSchema = valita_exports.object({
 	name: valita_exports.string()
 });
 var backfillIDSchema = jsonObjectSchema;
+var backfillProgressMarkSchema = valita_exports.object({
+	progressMark: valita_exports.string(),
+	timeline: valita_exports.string().optional()
+}).rest(valita_exports.union(valita_exports.number(), valita_exports.string()));
 var createTableSchema = valita_exports.object({
 	tag: valita_exports.literal("create-table"),
 	spec: tableSpec,
@@ -140,6 +144,10 @@ var backfillSchema = valita_exports.object({
 	columns: valita_exports.array(valita_exports.string()),
 	watermark: valita_exports.string(),
 	rowValues: valita_exports.array(valita_exports.array(jsonValueSchema)),
+	progressMarks: valita_exports.object({
+		previous: backfillProgressMarkSchema.optional(),
+		current: backfillProgressMarkSchema
+	}).optional(),
 	status: downloadStatusSchema.optional()
 });
 var backfillCompletedSchema = valita_exports.object({
@@ -147,6 +155,7 @@ var backfillCompletedSchema = valita_exports.object({
 	relation: newRelationSchema,
 	columns: valita_exports.array(valita_exports.string()),
 	watermark: valita_exports.string(),
+	progressMarks: valita_exports.object({ previous: backfillProgressMarkSchema.optional() }).optional(),
 	status: downloadStatusSchema.optional()
 });
 var dataChangeSchema = valita_exports.union(insertSchema, updateSchema, deleteSchema, truncateSchema, backfillSchema);
@@ -181,6 +190,6 @@ function isDataChange(change) {
 	return dataChangeTagSet.has(change.tag);
 }
 //#endregion
-export { addColumnSchema, backfillCompletedSchema, backfillIDSchema, backfillSchema, beginSchema, commitSchema, createIndexSchema, createTableSchema, dataChangeSchema, deleteSchema, downloadStatusSchema, dropColumnSchema, dropIndexSchema, dropTableSchema, identifierSchema, insertSchema, isDataChange, isSchemaChange, newRelationSchema, relationSchema, renameTableSchema, rollbackSchema, rowSchema, schemaChangeSchema, tableMetadataSchema, truncateSchema, updateColumnSchema, updateSchema, updateTableMetadataSchema };
+export { addColumnSchema, backfillCompletedSchema, backfillIDSchema, backfillProgressMarkSchema, backfillSchema, beginSchema, commitSchema, createIndexSchema, createTableSchema, dataChangeSchema, deleteSchema, downloadStatusSchema, dropColumnSchema, dropIndexSchema, dropTableSchema, identifierSchema, insertSchema, isDataChange, isSchemaChange, newRelationSchema, relationSchema, renameTableSchema, rollbackSchema, rowSchema, schemaChangeSchema, tableMetadataSchema, truncateSchema, updateColumnSchema, updateSchema, updateTableMetadataSchema };
 
 //# sourceMappingURL=data.js.map

package/out/zero-cache/src/services/change-source/protocol/current/data.js.map

@@ -1 +1 @@
-{"version":3,"file":"data.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/data.ts"],"sourcesContent":["/**\n * Data plane messages encapsulate changes that are sent by ChangeSources,\n * forwarded / fanned out to subscribers by the ChangeStreamerService, and\n * stored in the Change DB for catchup of old subscribers.\n */\n\nimport {\n  jsonValueSchema,\n  type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../../shared/src/must.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {columnSpec, indexSpec, tableSpec} from '../../../../db/specs.ts';\nimport type {Satisfies} from '../../../../types/satisfies.ts';\nimport {jsonObjectSchema} from './json.ts';\nimport {schemaChangeTags} from './schema-change-tags.ts';\n\nexport const beginSchema = v.object({\n  tag: v.literal('begin'),\n  // The format of values of \"json\"-typed columns (e.g. \"JSON\" and \"JSONB\").\n  // - 'p' is for parsed JSON, which may include JSON values or JSON objects.\n  //   These values are parsed and stringified at every process boundary\n  //   between the change-source and the replica.\n  // - 's' is for stringified JSON. These values skip the parsing and\n  //   stringification, and are directly ferried to the replica as a JSON\n  //   string. For JSON values this improves performance by 20~25% in the\n  //   change-streamer and 25~30% in the replicator.\n  //\n  // If absent, the format is assumed to be 'p' (parsed JSON objects/values).\n  json: v.literalUnion('p', 's').optional(),\n\n  // Directs the change-streamer to skip the ACK for the corresponding commit.\n  skipAck: v.boolean().optional(),\n});\n\nexport const commitSchema = v.object({\n  tag: v.literal('commit'),\n\n  // Millisecond epoch at which the transaction committed upstream, as reported\n  // by the upstream database (e.g. the `commitTime` of the Postgres logical\n  // replication Commit message).\n  //\n  // This is the origin timestamp of the end-to-end serving lag measurement:\n  // it is carried through the replication stream to the ViewSyncer, which\n  // records `now - commitTimeMs` once the change has been poked to clients.\n  //\n  // Optional because it is absent from changes written to the Change DB by\n  // older versions, which are replayed verbatim during catchup, and from\n  // ChangeSources that do not report a commit time.\n  commitTimeMs: v.number().optional(),\n});\n\nexport const rollbackSchema = v.object({\n  tag: v.literal('rollback'),\n});\n\nconst rowKeySchema = v.object({\n  // The columns used to identify a row in insert, update, and delete changes.\n  columns: v.array(v.string()),\n\n  // An optional qualifier identifying how the key is chosen. Currently this\n  // is postgres-specific, describing the REPLICA IDENTITY, for which replica\n  // identity 'full' (FULL) is handled differently; the replicator handles\n  // these tables by extracting a row key from the full row based on the\n  // table's PRIMARY KEY or UNIQUE INDEX.\n  type: v.literalUnion('default', 'nothing', 'full', 'index').optional(),\n});\n\nexport const relationSchema = v\n  .object({\n    schema: v.string(),\n    name: v.string(),\n\n    // This will become required.\n    rowKey: rowKeySchema.optional(),\n\n    /** Deprecated: set the rowKey.columns instead. */\n    keyColumns: v.array(v.string()).optional(),\n    /** Deprecated: set the rowKey.columns instead. */\n    replicaIdentity: v\n      .literalUnion('default', 'nothing', 'full', 'index')\n      .optional(),\n  })\n  .map(rel => {\n    const {rowKey, ...rest} = rel;\n    if (rowKey) {\n      return {...rest, rowKey};\n    }\n    return {\n      ...rest,\n      rowKey: {\n        columns: must(rel.keyColumns),\n        type: rel.replicaIdentity,\n      },\n    };\n  });\n\n// The eventual fate of relationSchema\nexport const newRelationSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n\n  rowKey: rowKeySchema,\n});\n\n// TableMetadata contains table-related configuration that does not affect the\n// actual data in the table, but rather how the table's change messages are\n// handled. The is an opaque object that clients must track (and update) based\n// on `create-table`, `add-column`, and `table-update-metadata` messages, and\n// pass in BackfillRequests when there are columns to be backfilled.\n//\n// Note that the backfill-related change-source implementation does, however,\n// rely on the rowKey (columns) being specified in the message.\nexport const tableMetadataSchema = v\n  .object({rowKey: v.record(jsonValueSchema)})\n  .rest(jsonValueSchema);\n\nexport type TableMetadata = v.Infer<typeof tableMetadataSchema>;\n\nexport const rowSchema = v.record(jsonValueSchema);\n\nexport const insertSchema = v.object({\n  tag: v.literal('insert'),\n  relation: relationSchema,\n  new: rowSchema,\n});\n\nexport const updateSchema = v.object({\n  tag: v.literal('update'),\n  relation: relationSchema,\n  // `key` is present if the update changed the key of the row, or if the\n  // table's replicaIdentity === 'full'\n  key: rowSchema.nullable(),\n  // `new` is the full row (and not just the updated columns). This is\n  // necessary for \"catchup\" replication scenarios such as adding tables\n  // to a publication, or resharding.\n  new: rowSchema,\n});\n\nexport const deleteSchema = v.object({\n  tag: v.literal('delete'),\n  relation: relationSchema,\n  // key is the full row if replicaIdentity === 'full'\n  key: rowSchema,\n});\n\nexport const truncateSchema = v.object({\n  tag: v.literal('truncate'),\n  relations: v.array(relationSchema),\n});\n\nexport const identifierSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n});\n\nexport type Identifier = v.Infer<typeof identifierSchema>;\n\n// A BackfillID is an upstream specific stable identifier for a column\n// that needs backfilling. This id is used to ensure that the schema, table,\n// and column names of a requested backfill still match the original\n// underlying upstream ID.\n//\n// The change-streamer stores these IDs as opaque values while a column is\n// being backfilled, and initiates new change-source streams with the IDs\n// in order to restart backfills that did not complete in previous sessions.\nexport const backfillIDSchema = jsonObjectSchema;\n\nexport type BackfillID = v.Infer<typeof backfillIDSchema>;\n\nexport const createTableSchema = v.object({\n  tag: v.literal('create-table'),\n  spec: tableSpec,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  metadata: tableMetadataSchema.optional(),\n\n  // Indicate that columns of the table require backfilling. These columns\n  // should be created on the replica but not yet synced the clients.\n  //\n  // ## State Persistence\n  //\n  // To obviate the need for change-source implementations to persist state\n  // related to backfill progress, the change-source only tracks backfills\n  // **for the current session**. In the event that the session is interrupted\n  // before columns have been fully backfilled, it is the responsibility of the\n  // change-streamer to send {@link BackfillRequest}s when it reconnects.\n  //\n  // This means that the change-streamer must track and persist:\n  // * the backfill IDs of the columns requiring backfilling\n  // * the most current table metadata of the associated table(s)\n  //\n  // The change-streamer then uses this information to send backfill requests\n  // when it reconnects.\n  backfill: v.record(backfillIDSchema).optional(),\n});\n\nexport const renameTableSchema = v.object({\n  tag: v.literal('rename-table'),\n  old: identifierSchema,\n  new: identifierSchema,\n});\n\nexport const updateTableMetadataSchema = v.object({\n  tag: v.literal('update-table-metadata'),\n  table: identifierSchema,\n  old: tableMetadataSchema,\n  new: tableMetadataSchema,\n});\n\nconst columnSchema = v.object({\n  name: v.string(),\n  spec: columnSpec,\n});\n\nexport const addColumnSchema = v.object({\n  tag: v.literal('add-column'),\n  table: identifierSchema,\n  column: columnSchema,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  tableMetadata: tableMetadataSchema.optional(),\n\n  // See documentation for the `backfill` field of the `create-table` change.\n  backfill: backfillIDSchema.optional(),\n});\n\nexport const updateColumnSchema = v.object({\n  tag: v.literal('update-column'),\n  table: identifierSchema,\n  old: columnSchema,\n  new: columnSchema,\n});\n\nexport const dropColumnSchema = v.object({\n  tag: v.literal('drop-column'),\n  table: identifierSchema,\n  column: v.string(),\n});\n\nexport const dropTableSchema = v.object({\n  tag: v.literal('drop-table'),\n  id: identifierSchema,\n});\n\nexport const createIndexSchema = v.object({\n  tag: v.literal('create-index'),\n  spec: indexSpec,\n});\n\nexport const dropIndexSchema = v.object({\n  tag: v.literal('drop-index'),\n  id: identifierSchema,\n});\n\nexport const downloadStatusSchema = v.object({\n  rows: v.number(),\n  totalRows: v.number(),\n  totalBytes: v.number().optional(),\n});\n\nexport type DownloadStatus = v.Infer<typeof downloadStatusSchema>;\n\n// A batch of rows from a single table containing column values\n// to be backfilled.\nexport const backfillSchema = v.object({\n  tag: v.literal('backfill'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n\n  // A batch of row values, each row consisting of the `rowKey`\n  // values, followed by the `column` values, in the same order in which\n  // the column names appear in their respective fields, e.g.\n  //\n  // ```\n  // [\n  //   [...rowKeyValues, ...columnValues],  // row 1\n  //   [...rowKeyValues, ...columnValues],  // row 2\n  // ]\n  // ```\n  rowValues: v.array(v.array(jsonValueSchema)),\n\n  // Optionally includes the progress of the backfill operation,\n  // for display purposes.\n  status: downloadStatusSchema.optional(),\n});\n\n// Indicates that the backfill for the specified columns have\n// been successfully backfilled and can be published to clients.\nexport const backfillCompletedSchema = v.object({\n  tag: v.literal('backfill-completed'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n\n  // Optionally includes the final status of the backfill operation,\n  // for display purposes.\n  status: downloadStatusSchema.optional(),\n});\n\nexport type MessageBegin = v.Infer<typeof beginSchema>;\nexport type MessageCommit = v.Infer<typeof commitSchema>;\nexport type MessageRollback = v.Infer<typeof rollbackSchema>;\n\nexport type MessageRelation = v.Infer<typeof relationSchema>;\nexport type MessageInsert = v.Infer<typeof insertSchema>;\nexport type MessageUpdate = v.Infer<typeof updateSchema>;\nexport type MessageDelete = v.Infer<typeof deleteSchema>;\nexport type MessageTruncate = v.Infer<typeof truncateSchema>;\n\nexport type MessageBackfill = v.Infer<typeof backfillSchema>;\n\nexport type TableCreate = v.Infer<typeof createTableSchema>;\nexport type TableRename = v.Infer<typeof renameTableSchema>;\nexport type TableUpdateMetadata = v.Infer<typeof updateTableMetadataSchema>;\nexport type ColumnAdd = v.Infer<typeof addColumnSchema>;\nexport type ColumnUpdate = v.Infer<typeof updateColumnSchema>;\nexport type ColumnDrop = v.Infer<typeof dropColumnSchema>;\nexport type TableDrop = v.Infer<typeof dropTableSchema>;\nexport type IndexCreate = v.Infer<typeof createIndexSchema>;\nexport type IndexDrop = v.Infer<typeof dropIndexSchema>;\nexport type BackfillCompleted = v.Infer<typeof backfillCompletedSchema>;\n\nexport const dataChangeSchema = v.union(\n  insertSchema,\n  updateSchema,\n  deleteSchema,\n  truncateSchema,\n  backfillSchema,\n);\n\n// Note: keep in sync or the tag tests will fail\nconst dataChangeTags = [\n  'insert',\n  'update',\n  'delete',\n  'truncate',\n  'backfill',\n] as const;\n\nconst dataChangeTagsSchema = v.literalUnion(...dataChangeTags);\n\nexport type DataChange = Satisfies<\n  JSONObject, // guarantees serialization over IPC or network\n  v.Infer<typeof dataChangeSchema>\n>;\n\nexport type DataChangeTag = v.Infer<typeof dataChangeTagsSchema>;\n\nconst schemaChanges = [\n  createTableSchema,\n  renameTableSchema,\n  updateTableMetadataSchema,\n  addColumnSchema,\n  updateColumnSchema,\n  dropColumnSchema,\n  dropTableSchema,\n  createIndexSchema,\n  dropIndexSchema,\n  backfillCompletedSchema,\n] as const;\n\nexport const schemaChangeSchema = v.union(...schemaChanges);\n\nconst schemaChangeTagsSchema = v.literalUnion(...schemaChangeTags);\n\nexport type SchemaChange = Satisfies<\n  JSONObject,\n  v.Infer<typeof schemaChangeSchema>\n>;\n\nexport type SchemaChangeTag = v.Infer<typeof schemaChangeTagsSchema>;\n\nexport type DataOrSchemaChange = DataChange | SchemaChange;\n\nexport type Change =\n  | MessageBegin\n  | DataOrSchemaChange\n  | MessageCommit\n  | MessageRollback;\n\nexport type ChangeTag = Change['tag'];\n\nconst schemaChangeTagSet = new Set<string>(schemaChangeTags);\n\nexport function isSchemaChange(change: Change): change is SchemaChange {\n  return schemaChangeTagSet.has(change.tag);\n}\n\nconst dataChangeTagSet = new Set<string>(dataChangeTags);\n\nexport function isDataChange(change: Change): change is DataChange {\n  return dataChangeTagSet.has(change.tag);\n}\n"],"mappings":";;;;;;;;;;;;AAiBA,IAAa,cAAc,eAAE,OAAO;CAClC,KAAK,eAAE,QAAQ,OAAO;CAWtB,MAAM,aAAe,KAAK,GAAG,EAAE,SAAS;CAGxC,SAAS,eAAE,QAAQ,EAAE,SAAS;AAChC,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAavB,cAAc,eAAE,OAAO,EAAE,SAAS;AACpC,CAAC;AAED,IAAa,iBAAiB,eAAE,OAAO,EACrC,KAAK,eAAE,QAAQ,UAAU,EAC3B,CAAC;AAED,IAAM,eAAe,eAAE,OAAO;CAE5B,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAO3B,MAAM,aAAe,WAAW,WAAW,QAAQ,OAAO,EAAE,SAAS;AACvE,CAAC;AAED,IAAa,iBAAiB,eAC3B,OAAO;CACN,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;CAGf,QAAQ,aAAa,SAAS;;CAG9B,YAAY,eAAE,MAAM,eAAE,OAAO,CAAC,EAAE,SAAS;;CAEzC,iBAAiB,aACD,WAAW,WAAW,QAAQ,OAAO,EAClD,SAAS;AACd,CAAC,EACA,KAAI,QAAO;CACV,MAAM,EAAC,QAAQ,GAAG,SAAQ;CAC1B,IAAI,QACF,OAAO;EAAC,GAAG;EAAM;CAAM;CAEzB,OAAO;EACL,GAAG;EACH,QAAQ;GACN,SAAS,KAAK,IAAI,UAAU;GAC5B,MAAM,IAAI;EACZ;CACF;AACF,CAAC;AAGH,IAAa,oBAAoB,eAAE,OAAO;CACxC,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;CAEf,QAAQ;AACV,CAAC;AAUD,IAAa,sBAAsB,eAChC,OAAO,EAAC,QAAQ,eAAE,OAAO,eAAe,EAAC,CAAC,EAC1C,KAAK,eAAe;AAIvB,IAAa,YAAY,eAAE,OAAO,eAAe;AAEjD,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CACV,KAAK;AACP,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CAGV,KAAK,UAAU,SAAS;CAIxB,KAAK;AACP,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CAEV,KAAK;AACP,CAAC;AAED,IAAa,iBAAiB,eAAE,OAAO;CACrC,KAAK,eAAE,QAAQ,UAAU;CACzB,WAAW,eAAE,MAAM,cAAc;AACnC,CAAC;AAED,IAAa,mBAAmB,eAAE,OAAO;CACvC,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;AACjB,CAAC;AAYD,IAAa,mBAAmB;AAIhC,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,MAAM;CAMN,UAAU,oBAAoB,SAAS;CAmBvC,UAAU,eAAE,OAAO,gBAAgB,EAAE,SAAS;AAChD,CAAC;AAED,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAa,4BAA4B,eAAE,OAAO;CAChD,KAAK,eAAE,QAAQ,uBAAuB;CACtC,OAAO;CACP,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAM,eAAe,eAAE,OAAO;CAC5B,MAAM,eAAE,OAAO;CACf,MAAM;AACR,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,OAAO;CACP,QAAQ;CAMR,eAAe,oBAAoB,SAAS;CAG5C,UAAU,iBAAiB,SAAS;AACtC,CAAC;AAED,IAAa,qBAAqB,eAAE,OAAO;CACzC,KAAK,eAAE,QAAQ,eAAe;CAC9B,OAAO;CACP,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAa,mBAAmB,eAAE,OAAO;CACvC,KAAK,eAAE,QAAQ,aAAa;CAC5B,OAAO;CACP,QAAQ,eAAE,OAAO;AACnB,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,IAAI;AACN,CAAC;AAED,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,MAAM;AACR,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,IAAI;AACN,CAAC;AAED,IAAa,uBAAuB,eAAE,OAAO;CAC3C,MAAM,eAAE,OAAO;CACf,WAAW,eAAE,OAAO;CACpB,YAAY,eAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAMD,IAAa,iBAAiB,eAAE,OAAO;CACrC,KAAK,eAAE,QAAQ,UAAU;CAEzB,UAAU;CAIV,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAM3B,WAAW,eAAE,OAAO;CAYpB,WAAW,eAAE,MAAM,eAAE,MAAM,eAAe,CAAC;CAI3C,QAAQ,qBAAqB,SAAS;AACxC,CAAC;AAID,IAAa,0BAA0B,eAAE,OAAO;CAC9C,KAAK,eAAE,QAAQ,oBAAoB;CAEnC,UAAU;CAIV,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAM3B,WAAW,eAAE,OAAO;CAIpB,QAAQ,qBAAqB,SAAS;AACxC,CAAC;AAyBD,IAAa,mBAAmB,eAAE,MAChC,cACA,cACA,cACA,gBACA,cACF;AAGA,IAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;AACF;AAE6B,aAAe,GAAG,cAAc;AAS7D,IAAM,gBAAgB;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,IAAa,qBAAqB,eAAE,MAAM,GAAG,aAAa;AAE3B,aAAe,GAAG,gBAAgB;AAmBjE,IAAM,qBAAqB,IAAI,IAAY,gBAAgB;AAE3D,SAAgB,eAAe,QAAwC;CACrE,OAAO,mBAAmB,IAAI,OAAO,GAAG;AAC1C;AAEA,IAAM,mBAAmB,IAAI,IAAY,cAAc;AAEvD,SAAgB,aAAa,QAAsC;CACjE,OAAO,iBAAiB,IAAI,OAAO,GAAG;AACxC"}
+{"version":3,"file":"data.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/data.ts"],"sourcesContent":["/**\n * Data plane messages encapsulate changes that are sent by ChangeSources,\n * forwarded / fanned out to subscribers by the ChangeStreamerService, and\n * stored in the Change DB for catchup of old subscribers.\n */\n\nimport {\n  jsonValueSchema,\n  type JSONObject,\n} from '../../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../../shared/src/must.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {columnSpec, indexSpec, tableSpec} from '../../../../db/specs.ts';\nimport type {Satisfies} from '../../../../types/satisfies.ts';\nimport {jsonObjectSchema} from './json.ts';\nimport {schemaChangeTags} from './schema-change-tags.ts';\n\nexport const beginSchema = v.object({\n  tag: v.literal('begin'),\n  // The format of values of \"json\"-typed columns (e.g. \"JSON\" and \"JSONB\").\n  // - 'p' is for parsed JSON, which may include JSON values or JSON objects.\n  //   These values are parsed and stringified at every process boundary\n  //   between the change-source and the replica.\n  // - 's' is for stringified JSON. These values skip the parsing and\n  //   stringification, and are directly ferried to the replica as a JSON\n  //   string. For JSON values this improves performance by 20~25% in the\n  //   change-streamer and 25~30% in the replicator.\n  //\n  // If absent, the format is assumed to be 'p' (parsed JSON objects/values).\n  json: v.literalUnion('p', 's').optional(),\n\n  // Directs the change-streamer to skip the ACK for the corresponding commit.\n  skipAck: v.boolean().optional(),\n});\n\nexport const commitSchema = v.object({\n  tag: v.literal('commit'),\n\n  // Millisecond epoch at which the transaction committed upstream, as reported\n  // by the upstream database (e.g. the `commitTime` of the Postgres logical\n  // replication Commit message).\n  //\n  // This is the origin timestamp of the end-to-end serving lag measurement:\n  // it is carried through the replication stream to the ViewSyncer, which\n  // records `now - commitTimeMs` once the change has been poked to clients.\n  //\n  // Optional because it is absent from changes written to the Change DB by\n  // older versions, which are replayed verbatim during catchup, and from\n  // ChangeSources that do not report a commit time.\n  commitTimeMs: v.number().optional(),\n});\n\nexport const rollbackSchema = v.object({\n  tag: v.literal('rollback'),\n});\n\nconst rowKeySchema = v.object({\n  // The columns used to identify a row in insert, update, and delete changes.\n  columns: v.array(v.string()),\n\n  // An optional qualifier identifying how the key is chosen. Currently this\n  // is postgres-specific, describing the REPLICA IDENTITY, for which replica\n  // identity 'full' (FULL) is handled differently; the replicator handles\n  // these tables by extracting a row key from the full row based on the\n  // table's PRIMARY KEY or UNIQUE INDEX.\n  type: v.literalUnion('default', 'nothing', 'full', 'index').optional(),\n});\n\nexport const relationSchema = v\n  .object({\n    schema: v.string(),\n    name: v.string(),\n\n    // This will become required.\n    rowKey: rowKeySchema.optional(),\n\n    /** Deprecated: set the rowKey.columns instead. */\n    keyColumns: v.array(v.string()).optional(),\n    /** Deprecated: set the rowKey.columns instead. */\n    replicaIdentity: v\n      .literalUnion('default', 'nothing', 'full', 'index')\n      .optional(),\n  })\n  .map(rel => {\n    const {rowKey, ...rest} = rel;\n    if (rowKey) {\n      return {...rest, rowKey};\n    }\n    return {\n      ...rest,\n      rowKey: {\n        columns: must(rel.keyColumns),\n        type: rel.replicaIdentity,\n      },\n    };\n  });\n\n// The eventual fate of relationSchema\nexport const newRelationSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n\n  rowKey: rowKeySchema,\n});\n\n// TableMetadata contains table-related configuration that does not affect the\n// actual data in the table, but rather how the table's change messages are\n// handled. The is an opaque object that clients must track (and update) based\n// on `create-table`, `add-column`, and `table-update-metadata` messages, and\n// pass in BackfillRequests when there are columns to be backfilled.\n//\n// Note that the backfill-related change-source implementation does, however,\n// rely on the rowKey (columns) being specified in the message.\nexport const tableMetadataSchema = v\n  .object({rowKey: v.record(jsonValueSchema)})\n  .rest(jsonValueSchema);\n\nexport type TableMetadata = v.Infer<typeof tableMetadataSchema>;\n\nexport const rowSchema = v.record(jsonValueSchema);\n\nexport const insertSchema = v.object({\n  tag: v.literal('insert'),\n  relation: relationSchema,\n  new: rowSchema,\n});\n\nexport const updateSchema = v.object({\n  tag: v.literal('update'),\n  relation: relationSchema,\n  // `key` is present if the update changed the key of the row, or if the\n  // table's replicaIdentity === 'full'\n  key: rowSchema.nullable(),\n  // `new` is the full row (and not just the updated columns). This is\n  // necessary for \"catchup\" replication scenarios such as adding tables\n  // to a publication, or resharding.\n  new: rowSchema,\n});\n\nexport const deleteSchema = v.object({\n  tag: v.literal('delete'),\n  relation: relationSchema,\n  // key is the full row if replicaIdentity === 'full'\n  key: rowSchema,\n});\n\nexport const truncateSchema = v.object({\n  tag: v.literal('truncate'),\n  relations: v.array(relationSchema),\n});\n\nexport const identifierSchema = v.object({\n  schema: v.string(),\n  name: v.string(),\n});\n\nexport type Identifier = v.Infer<typeof identifierSchema>;\n\n// A BackfillID is an upstream specific stable identifier for a column\n// that needs backfilling. This id is used to ensure that the schema, table,\n// and column names of a requested backfill still match the original\n// underlying upstream ID.\n//\n// The change-streamer stores these IDs as opaque values while a column is\n// being backfilled, and initiates new change-source streams with the IDs\n// in order to restart backfills that did not complete in previous sessions.\nexport const backfillIDSchema = jsonObjectSchema;\n\nexport type BackfillID = v.Infer<typeof backfillIDSchema>;\n\nexport const backfillProgressMarkSchema = v\n  .object({\n    // A lexicographically sortable string delineating the progress of a\n    // backfill. This is used to resume a backfill that did not complete.\n    progressMark: v.string(),\n\n    // Optionally, a timeline with which the progressMark is associated.\n    // A timeline is an opaque string that determines if backfill streams\n    // are \"compatible\" with each other, i.e. can be merged in a way that\n    // avoids gaps. progressMarks from different timelines are not\n    // compatible. If two timelines differ, a new backfill is started\n    // from scratch in order to get all subscribers on the same timeline.\n    //\n    // Timeline changes should be rare events (e.g. Postgres VACUUM FULL).\n    timeline: v.string().optional(),\n  })\n  // All other fields should be preserved and returned in BackfillRequests.\n  .rest(v.union(v.number(), v.string()));\n\nexport type BackfillProgressMark = v.Infer<typeof backfillProgressMarkSchema>;\n\nexport const createTableSchema = v.object({\n  tag: v.literal('create-table'),\n  spec: tableSpec,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  metadata: tableMetadataSchema.optional(),\n\n  // Indicate that columns of the table require backfilling. These columns\n  // should be created on the replica but not yet synced the clients.\n  //\n  // ## State Persistence\n  //\n  // To obviate the need for change-source implementations to persist state\n  // related to backfill progress, the change-source only tracks backfills\n  // **for the current session**. In the event that the session is interrupted\n  // before columns have been fully backfilled, it is the responsibility of the\n  // change-streamer to send {@link BackfillRequest}s when it reconnects.\n  //\n  // This means that the change-streamer must track and persist:\n  // * the backfill IDs of the columns requiring backfilling\n  // * the most current table metadata of the associated table(s)\n  //\n  // The change-streamer then uses this information to send backfill requests\n  // when it reconnects.\n  backfill: v.record(backfillIDSchema).optional(),\n});\n\nexport const renameTableSchema = v.object({\n  tag: v.literal('rename-table'),\n  old: identifierSchema,\n  new: identifierSchema,\n});\n\nexport const updateTableMetadataSchema = v.object({\n  tag: v.literal('update-table-metadata'),\n  table: identifierSchema,\n  old: tableMetadataSchema,\n  new: tableMetadataSchema,\n});\n\nconst columnSchema = v.object({\n  name: v.string(),\n  spec: columnSpec,\n});\n\nexport const addColumnSchema = v.object({\n  tag: v.literal('add-column'),\n  table: identifierSchema,\n  column: columnSchema,\n\n  // This must be set by change source implementations that support\n  // table/column backfill.\n  //\n  // TODO: to simplify the protocol, see if we can make this required\n  tableMetadata: tableMetadataSchema.optional(),\n\n  // See documentation for the `backfill` field of the `create-table` change.\n  backfill: backfillIDSchema.optional(),\n});\n\nexport const updateColumnSchema = v.object({\n  tag: v.literal('update-column'),\n  table: identifierSchema,\n  old: columnSchema,\n  new: columnSchema,\n});\n\nexport const dropColumnSchema = v.object({\n  tag: v.literal('drop-column'),\n  table: identifierSchema,\n  column: v.string(),\n});\n\nexport const dropTableSchema = v.object({\n  tag: v.literal('drop-table'),\n  id: identifierSchema,\n});\n\nexport const createIndexSchema = v.object({\n  tag: v.literal('create-index'),\n  spec: indexSpec,\n});\n\nexport const dropIndexSchema = v.object({\n  tag: v.literal('drop-index'),\n  id: identifierSchema,\n});\n\nexport const downloadStatusSchema = v.object({\n  rows: v.number(),\n  totalRows: v.number(),\n  totalBytes: v.number().optional(),\n});\n\nexport type DownloadStatus = v.Infer<typeof downloadStatusSchema>;\n\n// A batch of rows from a single table containing column values\n// to be backfilled.\nexport const backfillSchema = v.object({\n  tag: v.literal('backfill'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n\n  // A batch of row values, each row consisting of the `rowKey`\n  // values, followed by the `column` values, in the same order in which\n  // the column names appear in their respective fields, e.g.\n  //\n  // ```\n  // [\n  //   [...rowKeyValues, ...columnValues],  // row 1\n  //   [...rowKeyValues, ...columnValues],  // row 2\n  // ]\n  // ```\n  rowValues: v.array(v.array(jsonValueSchema)),\n\n  // Progress marks indicating the position of this backfill message\n  // within the stream, for resumption / continuity enforcement.\n  //\n  // This should be considered non-optional as of protocol v8. It is only\n  // absent in messages from pre-v8 replication-managers (including messages\n  // persisted in a change-log by one), in which case subscribers accept the\n  // message unconditionally (i.e. the legacy behavior).\n  progressMarks: v\n    .object({\n      // The progress mark of the preceding backfill message, from which\n      // the current backfill message is a continuation. (This may be a\n      // \"logical\" continuation at which a new backfill stream is started when\n      // resuming from a previous stream.)\n      //\n      // For the first backfill message of a stream starting from scratch,\n      // this field is unset. When the field is unset, subscribers should\n      // accept the backfill message unconditionally (i.e. starting a\n      // backfill from scratch is always safe).\n      //\n      // If the field is set, subscribers should only accept the message if\n      // it is of the same timeline as that of its own tracked progressMark,\n      // and is contiguous with or earlier than its own progressMark.\n      //\n      // In other words, it should _ignore_ backfill messages from other\n      // timelines, or messages that create a gap; this can happen when\n      // receiving messages from before this subscriber's backfill requests\n      // are taken into account.\n      //\n      // Implementation note: The change-streamer applies this same policy\n      // (see `acceptBackfill()` in `protocol/backfill-progress.ts`) to track\n      // the backfill state of each subscriber, which it uses to determine the\n      // BackfillRequests with which to (re)start change streams.\n      previous: backfillProgressMarkSchema.optional(),\n\n      // The progressMark delineating the position of the last row in this\n      // message. In the steady state, this will be the `previous`\n      // progressMark of the next backfill message, unless a rewind is\n      // requested, or new timeline encountered.\n      current: backfillProgressMarkSchema,\n    })\n    .optional(),\n\n  // Optionally includes the progress of the backfill operation,\n  // for display purposes.\n  status: downloadStatusSchema.optional(),\n});\n\n// Indicates that the backfill for the specified columns have\n// been successfully backfilled and can be published to clients.\nexport const backfillCompletedSchema = v.object({\n  tag: v.literal('backfill-completed'),\n\n  relation: newRelationSchema,\n\n  // The columns to be backfilled. `rowKey` columns are automatically excluded,\n  // which means that this field may be empty.\n  columns: v.array(v.string()),\n\n  // The watermark at which the backfill data was queried. Note that this\n  // generally will be different from the commit watermarks of the main change\n  // stream, and in particular, the commit watermark of the backfill change's\n  // enclosing transaction.\n  watermark: v.string(),\n\n  // The progress mark of the last backfill message of the run, i.e. the\n  // `current` progressMark of the last `backfill` message, or the progressMark\n  // from which the run was resumed if it produced no `backfill` messages.\n  // Subscribers apply the same continuity policy as for `backfill` messages,\n  // and only consider the columns backfilled if the run is contiguous with\n  // their own progress.\n  //\n  // `previous` is only unset for a run that started from scratch and\n  // produced no rows (e.g. an empty table), which is always safe to accept.\n  //\n  // This should be considered non-optional as of protocol v8. It is only\n  // absent in messages from pre-v8 replication-managers, in which case\n  // subscribers accept the message unconditionally (i.e. legacy behavior).\n  progressMarks: v\n    .object({previous: backfillProgressMarkSchema.optional()})\n    .optional(),\n\n  // Optionally includes the final status of the backfill operation,\n  // for display purposes.\n  status: downloadStatusSchema.optional(),\n});\n\nexport type MessageBegin = v.Infer<typeof beginSchema>;\nexport type MessageCommit = v.Infer<typeof commitSchema>;\nexport type MessageRollback = v.Infer<typeof rollbackSchema>;\n\nexport type MessageRelation = v.Infer<typeof relationSchema>;\nexport type MessageInsert = v.Infer<typeof insertSchema>;\nexport type MessageUpdate = v.Infer<typeof updateSchema>;\nexport type MessageDelete = v.Infer<typeof deleteSchema>;\nexport type MessageTruncate = v.Infer<typeof truncateSchema>;\n\nexport type MessageBackfill = v.Infer<typeof backfillSchema>;\n\nexport type TableCreate = v.Infer<typeof createTableSchema>;\nexport type TableRename = v.Infer<typeof renameTableSchema>;\nexport type TableUpdateMetadata = v.Infer<typeof updateTableMetadataSchema>;\nexport type ColumnAdd = v.Infer<typeof addColumnSchema>;\nexport type ColumnUpdate = v.Infer<typeof updateColumnSchema>;\nexport type ColumnDrop = v.Infer<typeof dropColumnSchema>;\nexport type TableDrop = v.Infer<typeof dropTableSchema>;\nexport type IndexCreate = v.Infer<typeof createIndexSchema>;\nexport type IndexDrop = v.Infer<typeof dropIndexSchema>;\nexport type BackfillCompleted = v.Infer<typeof backfillCompletedSchema>;\n\nexport const dataChangeSchema = v.union(\n  insertSchema,\n  updateSchema,\n  deleteSchema,\n  truncateSchema,\n  backfillSchema,\n);\n\n// Note: keep in sync or the tag tests will fail\nconst dataChangeTags = [\n  'insert',\n  'update',\n  'delete',\n  'truncate',\n  'backfill',\n] as const;\n\nconst dataChangeTagsSchema = v.literalUnion(...dataChangeTags);\n\nexport type DataChange = Satisfies<\n  JSONObject, // guarantees serialization over IPC or network\n  v.Infer<typeof dataChangeSchema>\n>;\n\nexport type DataChangeTag = v.Infer<typeof dataChangeTagsSchema>;\n\nconst schemaChanges = [\n  createTableSchema,\n  renameTableSchema,\n  updateTableMetadataSchema,\n  addColumnSchema,\n  updateColumnSchema,\n  dropColumnSchema,\n  dropTableSchema,\n  createIndexSchema,\n  dropIndexSchema,\n  backfillCompletedSchema,\n] as const;\n\nexport const schemaChangeSchema = v.union(...schemaChanges);\n\nconst schemaChangeTagsSchema = v.literalUnion(...schemaChangeTags);\n\nexport type SchemaChange = Satisfies<\n  JSONObject,\n  v.Infer<typeof schemaChangeSchema>\n>;\n\nexport type SchemaChangeTag = v.Infer<typeof schemaChangeTagsSchema>;\n\nexport type DataOrSchemaChange = DataChange | SchemaChange;\n\nexport type Change =\n  | MessageBegin\n  | DataOrSchemaChange\n  | MessageCommit\n  | MessageRollback;\n\nexport type ChangeTag = Change['tag'];\n\nconst schemaChangeTagSet = new Set<string>(schemaChangeTags);\n\nexport function isSchemaChange(change: Change): change is SchemaChange {\n  return schemaChangeTagSet.has(change.tag);\n}\n\nconst dataChangeTagSet = new Set<string>(dataChangeTags);\n\nexport function isDataChange(change: Change): change is DataChange {\n  return dataChangeTagSet.has(change.tag);\n}\n"],"mappings":";;;;;;;;;;;;AAiBA,IAAa,cAAc,eAAE,OAAO;CAClC,KAAK,eAAE,QAAQ,OAAO;CAWtB,MAAM,aAAe,KAAK,GAAG,EAAE,SAAS;CAGxC,SAAS,eAAE,QAAQ,EAAE,SAAS;AAChC,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAavB,cAAc,eAAE,OAAO,EAAE,SAAS;AACpC,CAAC;AAED,IAAa,iBAAiB,eAAE,OAAO,EACrC,KAAK,eAAE,QAAQ,UAAU,EAC3B,CAAC;AAED,IAAM,eAAe,eAAE,OAAO;CAE5B,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAO3B,MAAM,aAAe,WAAW,WAAW,QAAQ,OAAO,EAAE,SAAS;AACvE,CAAC;AAED,IAAa,iBAAiB,eAC3B,OAAO;CACN,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;CAGf,QAAQ,aAAa,SAAS;;CAG9B,YAAY,eAAE,MAAM,eAAE,OAAO,CAAC,EAAE,SAAS;;CAEzC,iBAAiB,aACD,WAAW,WAAW,QAAQ,OAAO,EAClD,SAAS;AACd,CAAC,EACA,KAAI,QAAO;CACV,MAAM,EAAC,QAAQ,GAAG,SAAQ;CAC1B,IAAI,QACF,OAAO;EAAC,GAAG;EAAM;CAAM;CAEzB,OAAO;EACL,GAAG;EACH,QAAQ;GACN,SAAS,KAAK,IAAI,UAAU;GAC5B,MAAM,IAAI;EACZ;CACF;AACF,CAAC;AAGH,IAAa,oBAAoB,eAAE,OAAO;CACxC,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;CAEf,QAAQ;AACV,CAAC;AAUD,IAAa,sBAAsB,eAChC,OAAO,EAAC,QAAQ,eAAE,OAAO,eAAe,EAAC,CAAC,EAC1C,KAAK,eAAe;AAIvB,IAAa,YAAY,eAAE,OAAO,eAAe;AAEjD,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CACV,KAAK;AACP,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CAGV,KAAK,UAAU,SAAS;CAIxB,KAAK;AACP,CAAC;AAED,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CACvB,UAAU;CAEV,KAAK;AACP,CAAC;AAED,IAAa,iBAAiB,eAAE,OAAO;CACrC,KAAK,eAAE,QAAQ,UAAU;CACzB,WAAW,eAAE,MAAM,cAAc;AACnC,CAAC;AAED,IAAa,mBAAmB,eAAE,OAAO;CACvC,QAAQ,eAAE,OAAO;CACjB,MAAM,eAAE,OAAO;AACjB,CAAC;AAYD,IAAa,mBAAmB;AAIhC,IAAa,6BAA6B,eACvC,OAAO;CAGN,cAAc,eAAE,OAAO;CAUvB,UAAU,eAAE,OAAO,EAAE,SAAS;AAChC,CAAC,EAEA,KAAK,eAAE,MAAM,eAAE,OAAO,GAAG,eAAE,OAAO,CAAC,CAAC;AAIvC,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,MAAM;CAMN,UAAU,oBAAoB,SAAS;CAmBvC,UAAU,eAAE,OAAO,gBAAgB,EAAE,SAAS;AAChD,CAAC;AAED,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAa,4BAA4B,eAAE,OAAO;CAChD,KAAK,eAAE,QAAQ,uBAAuB;CACtC,OAAO;CACP,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAM,eAAe,eAAE,OAAO;CAC5B,MAAM,eAAE,OAAO;CACf,MAAM;AACR,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,OAAO;CACP,QAAQ;CAMR,eAAe,oBAAoB,SAAS;CAG5C,UAAU,iBAAiB,SAAS;AACtC,CAAC;AAED,IAAa,qBAAqB,eAAE,OAAO;CACzC,KAAK,eAAE,QAAQ,eAAe;CAC9B,OAAO;CACP,KAAK;CACL,KAAK;AACP,CAAC;AAED,IAAa,mBAAmB,eAAE,OAAO;CACvC,KAAK,eAAE,QAAQ,aAAa;CAC5B,OAAO;CACP,QAAQ,eAAE,OAAO;AACnB,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,IAAI;AACN,CAAC;AAED,IAAa,oBAAoB,eAAE,OAAO;CACxC,KAAK,eAAE,QAAQ,cAAc;CAC7B,MAAM;AACR,CAAC;AAED,IAAa,kBAAkB,eAAE,OAAO;CACtC,KAAK,eAAE,QAAQ,YAAY;CAC3B,IAAI;AACN,CAAC;AAED,IAAa,uBAAuB,eAAE,OAAO;CAC3C,MAAM,eAAE,OAAO;CACf,WAAW,eAAE,OAAO;CACpB,YAAY,eAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAMD,IAAa,iBAAiB,eAAE,OAAO;CACrC,KAAK,eAAE,QAAQ,UAAU;CAEzB,UAAU;CAIV,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAM3B,WAAW,eAAE,OAAO;CAYpB,WAAW,eAAE,MAAM,eAAE,MAAM,eAAe,CAAC;CAS3C,eAAe,eACZ,OAAO;EAwBN,UAAU,2BAA2B,SAAS;EAM9C,SAAS;CACX,CAAC,EACA,SAAS;CAIZ,QAAQ,qBAAqB,SAAS;AACxC,CAAC;AAID,IAAa,0BAA0B,eAAE,OAAO;CAC9C,KAAK,eAAE,QAAQ,oBAAoB;CAEnC,UAAU;CAIV,SAAS,eAAE,MAAM,eAAE,OAAO,CAAC;CAM3B,WAAW,eAAE,OAAO;CAepB,eAAe,eACZ,OAAO,EAAC,UAAU,2BAA2B,SAAS,EAAC,CAAC,EACxD,SAAS;CAIZ,QAAQ,qBAAqB,SAAS;AACxC,CAAC;AAyBD,IAAa,mBAAmB,eAAE,MAChC,cACA,cACA,cACA,gBACA,cACF;AAGA,IAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;AACF;AAE6B,aAAe,GAAG,cAAc;AAS7D,IAAM,gBAAgB;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,IAAa,qBAAqB,eAAE,MAAM,GAAG,aAAa;AAE3B,aAAe,GAAG,gBAAgB;AAmBjE,IAAM,qBAAqB,IAAI,IAAY,gBAAgB;AAE3D,SAAgB,eAAe,QAAwC;CACrE,OAAO,mBAAmB,IAAI,OAAO,GAAG;AAC1C;AAEA,IAAM,mBAAmB,IAAI,IAAY,cAAc;AAEvD,SAAgB,aAAa,QAAsC;CACjE,OAAO,iBAAiB,IAAI,OAAO,GAAG;AACxC"}

package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts

@@ -31,7 +31,7 @@ declare const data: v.TupleType<[v.Type<"data">, v.UnionType<[v.UnionType<[v.Obj
             type?: "default" | "full" | "index" | "nothing" | undefined;
         };
     }>;
-    key: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue> | null>;
+    key: v.UnionType<[v.Type<null>, v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>]>;
     new: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>;
 }, undefined>, v.ObjectType<{
     tag: v.Type<"delete">;
@@ -71,6 +71,18 @@ declare const data: v.TupleType<[v.Type<"data">, v.UnionType<[v.UnionType<[v.Obj
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
     rowValues: v.ArrayType<v.ArrayType<v.Type<import("../../../../../../shared/src/bigint-json.ts").JSONValue>>>;
+    progressMarks: v.Optional<{
+        previous?: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        } | undefined;
+        current: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        };
+    }>;
     status: v.Optional<{
         rows: number;
         totalRows: number;
@@ -215,6 +227,13 @@ declare const data: v.TupleType<[v.Type<"data">, v.UnionType<[v.UnionType<[v.Obj
     }, undefined>;
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
+    progressMarks: v.Optional<{
+        previous?: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        } | undefined;
+    }>;
     status: v.Optional<{
         rows: number;
         totalRows: number;
@@ -265,7 +284,7 @@ export declare const changeStreamDataSchema: v.UnionType<[v.TupleType<[v.Type<"b
             type?: "default" | "full" | "index" | "nothing" | undefined;
         };
     }>;
-    key: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue> | null>;
+    key: v.UnionType<[v.Type<null>, v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>]>;
     new: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>;
 }, undefined>, v.ObjectType<{
     tag: v.Type<"delete">;
@@ -305,6 +324,18 @@ export declare const changeStreamDataSchema: v.UnionType<[v.TupleType<[v.Type<"b
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
     rowValues: v.ArrayType<v.ArrayType<v.Type<import("../../../../../../shared/src/bigint-json.ts").JSONValue>>>;
+    progressMarks: v.Optional<{
+        previous?: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        } | undefined;
+        current: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        };
+    }>;
     status: v.Optional<{
         rows: number;
         totalRows: number;
@@ -449,6 +480,13 @@ export declare const changeStreamDataSchema: v.UnionType<[v.TupleType<[v.Type<"b
     }, undefined>;
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
+    progressMarks: v.Optional<{
+        previous?: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        } | undefined;
+    }>;
     status: v.Optional<{
         rows: number;
         totalRows: number;
@@ -501,7 +539,7 @@ export declare const changeStreamMessageSchema: v.UnionType<[v.UnionType<[v.Tupl
             type?: "default" | "full" | "index" | "nothing" | undefined;
         };
     }>;
-    key: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue> | null>;
+    key: v.UnionType<[v.Type<null>, v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>]>;
     new: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>;
 }, undefined>, v.ObjectType<{
     tag: v.Type<"delete">;
@@ -541,6 +579,18 @@ export declare const changeStreamMessageSchema: v.UnionType<[v.UnionType<[v.Tupl
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
     rowValues: v.ArrayType<v.ArrayType<v.Type<import("../../../../../../shared/src/bigint-json.ts").JSONValue>>>;
+    progressMarks: v.Optional<{
+        previous?: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        } | undefined;
+        current: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        };
+    }>;
     status: v.Optional<{
         rows: number;
         totalRows: number;
@@ -685,6 +735,13 @@ export declare const changeStreamMessageSchema: v.UnionType<[v.UnionType<[v.Tupl
     }, undefined>;
     columns: v.ArrayType<v.Type<string>>;
     watermark: v.Type<string>;
+    progressMarks: v.Optional<{
+        previous?: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        } | undefined;
+    }>;
     status: v.Optional<{
         rows: number;
         totalRows: number;

package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"downstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/downstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAW5D,QAAA,MAAM,KAAK;;;;;;eAIT,CAAC;AACH,QAAA,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAGR,CAAC;AACH,QAAA,MAAM,MAAM;;;;;eAIV,CAAC;AACH,QAAA,MAAM,QAAQ;;eAAmD,CAAC;AAElE,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,CAAC;AAC1C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;AACxC,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAC5C,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,QAAQ,CAAC,CAAC;AAEhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAAyC,CAAC;AAC7E,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,yBAAyB;;;;eAGpC,CAAC;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,4EAA4E;AAC5E,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIrC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}
+{"version":3,"file":"downstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/downstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAW5D,QAAA,MAAM,KAAK;;;;;;eAIT,CAAC;AACH,QAAA,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAGR,CAAC;AACH,QAAA,MAAM,MAAM;;;;;eAIV,CAAC;AACH,QAAA,MAAM,QAAQ;;eAAmD,CAAC;AAElE,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,CAAC;AAC1C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;AACxC,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAC5C,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,QAAQ,CAAC,CAAC;AAEhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAAyC,CAAC;AAC7E,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,yBAAyB;;;;eAGpC,CAAC;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,4EAA4E;AAC5E,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIrC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts

@@ -22,24 +22,37 @@ export type ChangeSourceUpstream = v.Infer<typeof changeSourceUpstreamSchema>;
  * Backfills are automatically started for new tables and columns in a given
  * change stream session; however, if the session is terminated before the
  * backfill completes, it must be restarted with appropriate
- * {@link BackfillRequest}s when creating a new session.
+ * {@link BackfillRequest}s in subsequent session(s), as backfill state is
+ * ephemeral, per-replication-manager, and not persisted upstream.
  *
- * The `change-streamer` is responsible for tracking any changes to the table
- * name, column names, or table metadata, and constructing a BackfillRequest
- * based on the current values (which may be different from when the
- * tables/columns were originally added).
+ * All replication subscribers track their backfill state, which includes
+ * tracking any changes to the table name, column names, or table metadata,
+ * and present that state to the `change-streamer` when starting a
+ * subscription.
+ *
+ * From there the `change-streamer` manages requesting from the change-source
+ * the superset (and minimum progressMark) of all subscriber-specified
+ * backfills. While subscribers are connected, the change-streamer also tracks
+ * this per-subscriber state in memory, so that it can properly resume
+ * backfills on a new change-stream if disconnected from upstream.
  */
 export declare const backfillRequestSchema: v.ObjectType<{
     table: v.ObjectType<Omit<{
         schema: v.Type<string>;
         name: v.Type<string>;
     }, "metadata"> & {
-        metadata: v.Type<{
-            [x: string]: import("../../../../../../shared/src/bigint-json.ts").JSONValue;
-            rowKey: Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>;
-        } | null>;
+        metadata: v.UnionType<[v.Type<null>, v.ObjectType<{
+            rowKey: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>;
+        }, v.Type<import("../../../../../../shared/src/bigint-json.ts").JSONValue>>]>;
     }, undefined>;
-    columns: v.Type<Record<string, Record<string, import("./json.ts").JSONValue | undefined>>>;
+    columns: v.Type<Record<string, {
+        id: Record<string, import("./json.ts").JSONValue | undefined>;
+        progress?: {
+            [x: string]: string | number;
+            progressMark: string;
+            timeline?: string | undefined;
+        } | undefined;
+    }>>;
 }, undefined>;
 export type BackfillRequest = v.Infer<typeof backfillRequestSchema>;
 //# sourceMappingURL=upstream.d.ts.map

package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"upstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAQ5D,sEAAsE;AACtE,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;eAA8B,CAAC;AACtE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAE9E;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;aAOhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}
+{"version":3,"file":"upstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAS5D,sEAAsE;AACtE,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;eAA8B,CAAC;AACtE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAE9E;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;aAYhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/upstream.js

@@ -1,5 +1,5 @@
 import { valita_exports } from "../../../../../../shared/src/valita.js";
-import { backfillIDSchema, identifierSchema, tableMetadataSchema } from "./data.js";
+import { backfillIDSchema, backfillProgressMarkSchema, identifierSchema, tableMetadataSchema } from "./data.js";
 import { upstreamStatusMessageSchema } from "./status.js";
 //#region ../zero-cache/src/services/change-source/protocol/current/upstream.ts
 /** At the moment, the only upstream messages are status messages.  */
@@ -9,16 +9,26 @@ var changeSourceUpstreamSchema = upstreamStatusMessageSchema;
 * Backfills are automatically started for new tables and columns in a given
 * change stream session; however, if the session is terminated before the
 * backfill completes, it must be restarted with appropriate
-* {@link BackfillRequest}s when creating a new session.
+* {@link BackfillRequest}s in subsequent session(s), as backfill state is
+* ephemeral, per-replication-manager, and not persisted upstream.
 *
-* The `change-streamer` is responsible for tracking any changes to the table
-* name, column names, or table metadata, and constructing a BackfillRequest
-* based on the current values (which may be different from when the
-* tables/columns were originally added).
+* All replication subscribers track their backfill state, which includes
+* tracking any changes to the table name, column names, or table metadata,
+* and present that state to the `change-streamer` when starting a
+* subscription.
+*
+* From there the `change-streamer` manages requesting from the change-source
+* the superset (and minimum progressMark) of all subscriber-specified
+* backfills. While subscribers are connected, the change-streamer also tracks
+* this per-subscriber state in memory, so that it can properly resume
+* backfills on a new change-stream if disconnected from upstream.
 */
 var backfillRequestSchema = valita_exports.object({
 	table: identifierSchema.extend({ metadata: tableMetadataSchema.nullable() }),
-	columns: valita_exports.record(backfillIDSchema)
+	columns: valita_exports.record(valita_exports.object({
+		id: backfillIDSchema,
+		progress: backfillProgressMarkSchema.optional()
+	}))
 });
 //#endregion
 export { backfillRequestSchema, changeSourceUpstreamSchema };

package/out/zero-cache/src/services/change-source/protocol/current/upstream.js.map

@@ -1 +1 @@
-{"version":3,"file":"upstream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {\n  backfillIDSchema,\n  identifierSchema,\n  tableMetadataSchema,\n} from './data.ts';\nimport {upstreamStatusMessageSchema} from './status.ts';\n\n/** At the moment, the only upstream messages are status messages.  */\nexport const changeSourceUpstreamSchema = upstreamStatusMessageSchema;\nexport type ChangeSourceUpstream = v.Infer<typeof changeSourceUpstreamSchema>;\n\n/**\n * Contains the information for requesting a backfill of columns in a table.\n * Backfills are automatically started for new tables and columns in a given\n * change stream session; however, if the session is terminated before the\n * backfill completes, it must be restarted with appropriate\n * {@link BackfillRequest}s when creating a new session.\n *\n * The `change-streamer` is responsible for tracking any changes to the table\n * name, column names, or table metadata, and constructing a BackfillRequest\n * based on the current values (which may be different from when the\n * tables/columns were originally added).\n */\nexport const backfillRequestSchema = v.object({\n  table: identifierSchema.extend({\n    // The table metadata is set to null if it is never specified by the\n    // change-source.\n    metadata: tableMetadataSchema.nullable(),\n  }),\n  columns: v.record(backfillIDSchema),\n});\n\nexport type BackfillRequest = v.Infer<typeof backfillRequestSchema>;\n"],"mappings":";;;;;AASA,IAAa,6BAA6B;;;;;;;;;;;;;AAe1C,IAAa,wBAAwB,eAAE,OAAO;CAC5C,OAAO,iBAAiB,OAAO,EAG7B,UAAU,oBAAoB,SAAS,EACzC,CAAC;CACD,SAAS,eAAE,OAAO,gBAAgB;AACpC,CAAC"}
+{"version":3,"file":"upstream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {\n  backfillIDSchema,\n  backfillProgressMarkSchema,\n  identifierSchema,\n  tableMetadataSchema,\n} from './data.ts';\nimport {upstreamStatusMessageSchema} from './status.ts';\n\n/** At the moment, the only upstream messages are status messages.  */\nexport const changeSourceUpstreamSchema = upstreamStatusMessageSchema;\nexport type ChangeSourceUpstream = v.Infer<typeof changeSourceUpstreamSchema>;\n\n/**\n * Contains the information for requesting a backfill of columns in a table.\n * Backfills are automatically started for new tables and columns in a given\n * change stream session; however, if the session is terminated before the\n * backfill completes, it must be restarted with appropriate\n * {@link BackfillRequest}s in subsequent session(s), as backfill state is\n * ephemeral, per-replication-manager, and not persisted upstream.\n *\n * All replication subscribers track their backfill state, which includes\n * tracking any changes to the table name, column names, or table metadata,\n * and present that state to the `change-streamer` when starting a\n * subscription.\n *\n * From there the `change-streamer` manages requesting from the change-source\n * the superset (and minimum progressMark) of all subscriber-specified\n * backfills. While subscribers are connected, the change-streamer also tracks\n * this per-subscriber state in memory, so that it can properly resume\n * backfills on a new change-stream if disconnected from upstream.\n */\nexport const backfillRequestSchema = v.object({\n  table: identifierSchema.extend({\n    // The table metadata is set to null if it is never specified by the\n    // change-source.\n    metadata: tableMetadataSchema.nullable(),\n  }),\n  columns: v.record(\n    v.object({\n      id: backfillIDSchema,\n      progress: backfillProgressMarkSchema.optional(),\n    }),\n  ),\n});\n\nexport type BackfillRequest = v.Infer<typeof backfillRequestSchema>;\n"],"mappings":";;;;;AAUA,IAAa,6BAA6B;;;;;;;;;;;;;;;;;;;;AAsB1C,IAAa,wBAAwB,eAAE,OAAO;CAC5C,OAAO,iBAAiB,OAAO,EAG7B,UAAU,oBAAoB,SAAS,EACzC,CAAC;CACD,SAAS,eAAE,OACT,eAAE,OAAO;EACP,IAAI;EACJ,UAAU,2BAA2B,SAAS;CAChD,CAAC,CACH;AACF,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current.js

@@ -1,7 +1,7 @@
 import { __exportAll } from "../../../../../_virtual/_rolldown/runtime.js";
 import { jsonObjectSchema, jsonValueSchema } from "./current/json.js";
 import { resetRequiredSchema } from "./current/control.js";
-import { addColumnSchema, backfillCompletedSchema, backfillIDSchema, backfillSchema, beginSchema, commitSchema, createIndexSchema, createTableSchema, dataChangeSchema, deleteSchema, downloadStatusSchema, dropColumnSchema, dropIndexSchema, dropTableSchema, identifierSchema, insertSchema, isDataChange, isSchemaChange, newRelationSchema, relationSchema, renameTableSchema, rollbackSchema, rowSchema, schemaChangeSchema, tableMetadataSchema, truncateSchema, updateColumnSchema, updateSchema, updateTableMetadataSchema } from "./current/data.js";
+import { addColumnSchema, backfillCompletedSchema, backfillIDSchema, backfillProgressMarkSchema, backfillSchema, beginSchema, commitSchema, createIndexSchema, createTableSchema, dataChangeSchema, deleteSchema, downloadStatusSchema, dropColumnSchema, dropIndexSchema, dropTableSchema, identifierSchema, insertSchema, isDataChange, isSchemaChange, newRelationSchema, relationSchema, renameTableSchema, rollbackSchema, rowSchema, schemaChangeSchema, tableMetadataSchema, truncateSchema, updateColumnSchema, updateSchema, updateTableMetadataSchema } from "./current/data.js";
 import { downstreamStatusMessageSchema, downstreamStatusSchema, upstreamStatusMessageSchema } from "./current/status.js";
 import { changeStreamControlSchema, changeStreamDataSchema, changeStreamMessageSchema } from "./current/downstream.js";
 import { CHANGE_SOURCE_PATH } from "./current/path.js";
@@ -12,6 +12,7 @@ var current_exports = /* @__PURE__ */ __exportAll({
 	addColumnSchema: () => addColumnSchema,
 	backfillCompletedSchema: () => backfillCompletedSchema,
 	backfillIDSchema: () => backfillIDSchema,
+	backfillProgressMarkSchema: () => backfillProgressMarkSchema,
 	backfillRequestSchema: () => backfillRequestSchema,
 	backfillSchema: () => backfillSchema,
 	beginSchema: () => beginSchema,

package/out/zero-cache/src/services/change-streamer/backfill-state-tracker.d.ts

@@ -0,0 +1,72 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { Subscription } from '../../types/subscription.ts';
+import type { ChangeStreamData } from '../change-source/protocol/current/downstream.ts';
+import type { BackfillRequest } from '../change-source/protocol/current/upstream.ts';
+import type { PreSerializedBatch } from './broadcast.ts';
+import type { SubscriberContext } from './change-streamer.ts';
+import type { Subscriber, SubscriberOptions } from './subscriber.ts';
+export type SubscriberBackfillOptions = Pick<SubscriberOptions, 'backfills' | 'onAligned' | 'onBackfillIgnored'>;
+/**
+ * Tracks the pending backfills (and their progress) of the change log, the
+ * subscribers, and the current change stream, in order to determine:
+ *
+ * * the BackfillRequests with which to start a change stream, i.e. the
+ *   superset of the pending backfills of the change log and all aligned
+ *   subscribers, starting from the earliest progress, and
+ * * whether (and why) the change stream must be restarted, i.e. when a
+ *   newly aligned subscriber's pending backfills are not covered by the
+ *   current stream.
+ *
+ * The change-streamer calls {@link startStream()} whenever it starts a change
+ * stream, {@link track()} for every change that it forwards, and restarts
+ * the stream at the next transaction boundary whenever a
+ * {@link restartReason} is set.
+ */
+export declare class BackfillStateTracker {
+    #private;
+    constructor(lc: LogContext);
+    /**
+     * The reason for which the change stream must be restarted, or `null` if
+     * it need not be. The restart should happen at the next transaction
+     * boundary, i.e. upon the next `commit`, or `status` message outside of a
+     * transaction.
+     */
+    get restartReason(): string | null;
+    /**
+     * Computes the BackfillRequests with which to start a new change stream:
+     * the superset of the pending backfills of the change log and of all
+     * aligned subscribers, all of which are consistent with the stream's
+     * starting watermark since streams are (re)started at transaction
+     * boundaries.
+     *
+     * @param logBackfillRequests The BackfillRequests from the change log's
+     *        cookie jar, consistent with the stream's starting watermark.
+     */
+    startStream(logBackfillRequests: BackfillRequest[]): BackfillRequest[];
+    /**
+     * Tracks a change that is forwarded to subscribers. This must be called
+     * (synchronously) before the change is forwarded, so that the stream's
+     * state is never ahead of that of aligned subscribers.
+     */
+    track(change: ChangeStreamData): void;
+    /**
+     * Requests a restart of the change stream (at the next transaction
+     * boundary), e.g. to start it with BackfillRequests that it was not
+     * started with.
+     */
+    requestRestart(reason: string): void;
+    /** Logs and records a (planned) restart for the {@link restartReason}. */
+    recordRestart(): void;
+    /**
+     * Returns the backfill-related {@link SubscriberOptions} for a new
+     * subscriber, which must subsequently be passed to {@link register()}.
+     */
+    subscriberOptions(lc: LogContext, ctx: SubscriberContext): SubscriberBackfillOptions;
+    /**
+     * Registers a subscriber (created with {@link subscriberOptions()}) until
+     * its `downstream` is closed. This is a no-op for subscribers that do not
+     * report their pending backfills.
+     */
+    register(subscriber: Subscriber, downstream: Subscription<string | PreSerializedBatch>): void;
+}
+//# sourceMappingURL=backfill-state-tracker.d.ts.map