@intentius/chant-lexicon-sql 0.116.0 → 0.118.0

package/src/clickhouse/plan/normalize.ts

@@ -32,13 +32,14 @@
  */
 
 import { isTrivia, tokenizeText, type Token } from "../tokens";
-import { parseCreate, unquote, type ColumnNode, type CreateNode, type Span } from "../parser";
+import { parseCreate, unquote, type ColumnNode, type CreateNode, type DictionaryAttributeNode, type DictionaryNode, type Span } from "../parser";
 import { MERGE_TREE_SETTINGS, TYPE_FAMILIES } from "../../generated/clickhouse";
 import { previouslyIn, splitTopLevel as splitTop } from "../../core/normalize";
+import { accessCanonical, grantsCanonical, isAccessStatement } from "../access";
 
 export { previouslyIn } from "../../core/normalize";
 
-export type ObjectKind = "database" | "table" | "view" | "materializedView";
+export type ObjectKind = "database" | "table" | "view" | "materializedView" | "dictionary" | "function" | "user" | "role" | "rowPolicy" | "grants";
 
 export interface CanonicalColumn {
   name: string;
@@ -50,6 +51,10 @@ export interface CanonicalColumn {
   codec?: string;
   ttl?: string;
   comment?: string;
+  /** A dictionary attribute's `EXPRESSION`. */
+  expression?: string;
+  /** A dictionary attribute's flags, sorted: `HIERARCHICAL`, `INJECTIVE`. */
+  flags?: string;
   /** `-- previously: <name>` on the column's line: the name it had before. */
   previously?: string;
   /** The column's name and type as written, for a message. Not compared. */
@@ -77,6 +82,20 @@ export interface CanonicalObject {
   select?: string;
   to?: string;
   refresh?: string;
+  /** A dictionary's `SOURCE`, `LAYOUT`, `LIFETIME` and `RANGE`, each what is inside the parentheses. */
+  dataSource?: string;
+  layout?: string;
+  lifetime?: string;
+  range?: string;
+  /** A function's lambda, `( x , y ) -> expr`. */
+  lambda?: string;
+  /** A row policy's table. */
+  table?: string;
+  /**
+   * A user's, role's or row policy's fields (`../access.ts`), each absent at
+   * the server's default; a grantee's grants, by atom.
+   */
+  access?: Record<string, string>;
   /** `-- previously: <name>` before the statement: the object's previous name. */
   previously?: string;
 }
@@ -231,7 +250,7 @@ function lineComments(tokens: Token[], from: number): string[] {
 }
 
 const kindOf = (node: CreateNode): ObjectKind =>
-  node.statement === "database" ? "database" : node.statement === "table" ? "table" : node.materialized ? "materializedView" : "view";
+  node.statement === "view" ? (node.materialized ? "materializedView" : "view") : node.statement;
 
 const stringValue = (s: string | undefined) =>
   s === undefined ? undefined : /^'.*'$/s.test(s) ? s.slice(1, -1).replace(/''/g, "'").replace(/\\(.)/g, "$1") : s;
@@ -241,6 +260,22 @@ const stringValue = (s: string | undefined) =>
  * database an unqualified name is created in.
  */
 export function canonicalObject(ddl: string, defaultDatabase = "default"): CanonicalObject {
+  if (isAccessStatement(ddl)) {
+    if (/^\s*(?:--[^\n]*\n\s*)*(GRANT|REVOKE)\b/i.test(ddl)) throw new Error("a grant is compared as its grantee's grants: grantsObject(ddl, grantee)");
+    const a = accessCanonical(ddl, defaultDatabase);
+    return {
+      kind: a.kind,
+      ...(a.database !== undefined ? { database: a.database } : {}),
+      name: a.name,
+      ...(a.table !== undefined ? { table: a.table } : {}),
+      access: a.fields,
+      columns: [],
+      settings: {},
+      indexes: {},
+      projections: {},
+      constraints: {},
+    };
+  }
   const tokens = tokenizeText(ddl, 0);
   const node = parseCreate(tokens);
   const nameParts = tokens
@@ -249,7 +284,7 @@ export function canonicalObject(ddl: string, defaultDatabase = "default"): Canon
     .map((t) => unquote(t.text));
   const kind = kindOf(node);
   const name = nameParts[nameParts.length - 1]!;
-  const database = kind === "database" ? undefined : nameParts.length >= 2 ? nameParts[nameParts.length - 2] : defaultDatabase;
+  const database = kind === "database" || kind === "function" ? undefined : nameParts.length >= 2 ? nameParts[nameParts.length - 2] : defaultDatabase;
   const expr = (span: Span | undefined) => {
     const t = spanText(tokens, span);
     return t === undefined ? undefined : canonicalExpression(t, database);
@@ -269,6 +304,16 @@ export function canonicalObject(ddl: string, defaultDatabase = "default"): Canon
   const previously = previouslyIn(leading);
   if (previously) obj.previously = previously;
 
+  const comment0 = stringValue(spanText(tokens, node.comment));
+  if (node.statement === "dictionary") {
+    if (comment0 !== undefined && comment0 !== "") obj.comment = comment0;
+    return dictionaryObject(obj, tokens, node, database);
+  }
+  if (node.statement === "function") {
+    obj.lambda = canonicalExpression(spanText(tokens, node.lambda) ?? "");
+    return obj;
+  }
+
   const engine = node.engine;
   if (engine) {
     let args = engine.args?.map((a) => canonicalExpression(spanText(tokens, a) ?? "", database)) ?? [];
@@ -291,8 +336,7 @@ export function canonicalObject(ddl: string, defaultDatabase = "default"): Canon
     obj.engineName = name;
     obj.engine = args.length ? `${name}(${args.join(", ")})` : name;
   }
-  const comment = stringValue(spanText(tokens, node.comment));
-  if (comment !== undefined && comment !== "") obj.comment = comment;
+  if (comment0 !== undefined && comment0 !== "") obj.comment = comment0;
 
   if (node.statement === "database") {
     for (const s of node.settings ?? []) obj.settings[s.key] = canonicalExpression(spanText(tokens, s.value) ?? "").replace(/^'(.*)'$/, "$1");
@@ -333,6 +377,111 @@ export function canonicalObject(ddl: string, defaultDatabase = "default"): Canon
   return obj;
 }
 
+// ── dictionaries ──────────────────────────────────────────────────────
+
+/**
+ * A dictionary's clause (`SOURCE`, `LAYOUT`) as the server prints it: each
+ * function name and each key upper case (`clickhouse(table 'r')` is
+ * `CLICKHOUSE(TABLE 'r')`), each value as written, and a `PASSWORD` value
+ * as `'[HIDDEN]'`, since `SHOW CREATE` never prints it.
+ */
+export function canonicalDictionaryClause(text: string): string {
+  let sig: Token[];
+  try {
+    sig = tokenizeText(text, 0).filter((t) => !isTrivia(t));
+  } catch {
+    return text.trim();
+  }
+  const out: string[] = [];
+  let key = true;
+  let hide = false;
+  for (let i = 0; i < sig.length; i++) {
+    const t = sig[i]!;
+    const next = sig[i + 1];
+    if (t.kind === "punct" && (t.text === "(" || t.text === "," || t.text === ")")) {
+      out.push(t.text);
+      key = true;
+      continue;
+    }
+    if (key && t.kind === "ident") {
+      const word = t.text.toUpperCase();
+      out.push(word);
+      // A function name opens its own list of keys; a key is followed by its value.
+      key = next?.kind === "punct" && next.text === "(";
+      hide = word === "PASSWORD";
+      continue;
+    }
+    out.push(hide ? "'[HIDDEN]'" : t.kind === "qident" ? unquote(t.text) : t.text);
+    hide = false;
+    key = !(next?.kind === "punct" && next.text === "(");
+  }
+  return out.join(" ");
+}
+
+/** `300` is `MIN 0 MAX 300`; `MAX b MIN a` is `MIN a MAX b`. */
+export function canonicalLifetime(text: string): string {
+  const words = canonicalExpression(text).split(" ");
+  if (words.length === 1) return `MIN 0 MAX ${words[0]}`;
+  const at = (w: string) => words.findIndex((x) => x.toUpperCase() === w);
+  const min = at("MIN");
+  const max = at("MAX");
+  if (min < 0 || max < 0) return words.join(" ");
+  return `MIN ${words[min + 1] ?? ""} MAX ${words[max + 1] ?? ""}`;
+}
+
+/** A key list with any parentheses around the whole of it left out: `(a, b)` is `a , b`. */
+function keyList(text: string, database?: string): string {
+  const e = canonicalExpression(text, database);
+  if (!e.startsWith("( ") || !e.endsWith(" )")) return e;
+  let depth = 0;
+  const words = e.split(" ");
+  for (let i = 0; i < words.length; i++) {
+    if (words[i] === "(") depth++;
+    else if (words[i] === ")") depth--;
+    if (depth === 0 && i < words.length - 1) return e;
+  }
+  return words.slice(1, -1).join(" ");
+}
+
+function dictionaryAttribute(tokens: Token[], a: DictionaryAttributeNode, position: number, database?: string): CanonicalColumn {
+  const text = (span: Span | undefined) => spanText(tokens, span);
+  const out: CanonicalColumn = { name: a.name, position, type: canonicalType(text(a.type)!), text: [a.name, text(a.type)].join(" ") };
+  const d = text(a.default);
+  if (d !== undefined) {
+    out.defaultKind = "DEFAULT";
+    out.defaultExpr = canonicalExpression(d, database);
+  }
+  const e = text(a.expression);
+  if (e !== undefined) out.expression = canonicalExpression(e, database);
+  if (a.flags.length > 0) out.flags = [...a.flags].sort().join(" ");
+  const prev = previouslyIn(lineComments(tokens, a.nameSpan.from));
+  if (prev) out.previously = prev;
+  return out;
+}
+
+function dictionaryObject(obj: CanonicalObject, tokens: Token[], node: DictionaryNode, database?: string): CanonicalObject {
+  const text = (span: Span | undefined) => spanText(tokens, span);
+  node.attributes.forEach((a, i) => obj.columns.push(dictionaryAttribute(tokens, a, i, database)));
+  const pk = text(node.primaryKey);
+  if (pk !== undefined) obj.primaryKey = keyList(pk, database);
+  const source = text(node.source);
+  if (source !== undefined) obj.dataSource = canonicalDictionaryClause(source);
+  const layout = text(node.layout);
+  if (layout !== undefined) obj.layout = canonicalDictionaryClause(layout);
+  const lifetime = text(node.lifetime);
+  if (lifetime !== undefined) obj.lifetime = canonicalLifetime(lifetime);
+  const range = text(node.range);
+  if (range !== undefined) obj.range = canonicalExpression(range, database).replace(/\b(min|max)\b/gi, (w) => w.toUpperCase());
+  const settings = text(node.settings);
+  if (settings !== undefined) {
+    for (const item of splitTop(canonicalExpression(settings))) {
+      const m = /^(\S+) = (.*)$/.exec(item);
+      if (m) obj.settings[m[1]!] = m[2]!.replace(/^'(.*)'$/, "$1");
+    }
+  }
+  return obj;
+}
+
 function qualified(tokens: Token[], span: Span, defaultDatabase: string): string {
   const parts = tokens
     .slice(span.from, span.to)
@@ -369,3 +518,48 @@ function quotedIdentifier(arg: string): string {
   const bare = /^[A-Za-z_][A-Za-z0-9_]*$/.test(arg) ? arg : /^`((?:[^`]|``)+)`$/.exec(arg)?.[1]?.replace(/``/g, "`");
   return bare === undefined ? arg : `'${bare.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
 }
+
+/**
+ * The key an object is compared under against a server: `database.name`,
+ * a database's name, and `function <name>` for a function, which belongs to
+ * no database and must not meet a database of the same name.
+ */
+export function objectKey(o: { kind?: ObjectKind; type?: string; database?: string; name: string; table?: string }): string {
+  const kind = o.kind ?? KIND_OF_TYPE[o.type ?? ""];
+  if (kind === "function") return `function ${o.name}`;
+  if (kind === "user") return `user ${o.name}`;
+  if (kind === "role") return `role ${o.name}`;
+  if (kind === "grants") return `grants ${o.name}`;
+  if (kind === "rowPolicy") return `row policy ${o.name} ON ${o.database}.${o.table}`;
+  return o.database !== undefined ? `${o.database}.${o.name}` : o.name;
+}
+
+const KIND_OF_TYPE: Record<string, ObjectKind> = {
+  "ClickHouse::Function": "function",
+  "ClickHouse::User": "user",
+  "ClickHouse::Role": "role",
+  "ClickHouse::RowPolicy": "rowPolicy",
+  "ClickHouse::Grant": "grants",
+};
+
+/** A grantee's grants, from grant statements (one or more, on their own lines), as a plan compares them. */
+export function grantsObject(ddl: string, grantee: string, defaultDatabase = "default"): CanonicalObject {
+  return { kind: "grants", name: grantee, access: grantsCanonical(ddl, grantee, defaultDatabase), columns: [], settings: {}, indexes: {}, projections: {}, constraints: {} };
+}
+
+/**
+ * What an object is scoped by when a plan reads a server: its database, a
+ * database's own name, and for a function its key. A plan reads the
+ * databases the declarations use and the functions they declare, nothing
+ * else.
+ */
+export function scopeOf(o: { kind?: ObjectKind; type?: string; database?: string; name: string; table?: string }): string {
+  const kind = o.kind ?? KIND_OF_TYPE[o.type ?? ""];
+  if (kind === "function" || kind === "user" || kind === "role" || kind === "grants") return objectKey(o);
+  return o.database ?? o.name;
+}
+
+/** A live object's canonical form: a grantee's grants from its `SHOW GRANTS` lines, anything else from its `SHOW CREATE`. */
+export function liveCanonical(o: { type: string; name: string; statement: string }, defaultDatabase = "default"): CanonicalObject {
+  return o.type === "ClickHouse::Grant" ? grantsObject(o.statement, o.name, defaultDatabase) : canonicalObject(o.statement, defaultDatabase);
+}

package/src/clickhouse/plan/report.ts

@@ -6,14 +6,25 @@ import type { SchemaDiff } from "./diff";
 import { renderRebuildOps } from "./rebuild-handoff";
 
 export function renderDiff(diff: SchemaDiff, opts: { title?: string } = {}): string {
-  const trailer =
-    diff.rebuilds.length > 0
+  const unreadable = diff.unreadable ?? [];
+  const refusedRead =
+    unreadable.length > 0
+      ? [
+          "",
+          `Refused: ${unreadable.length} object(s) in the declared databases could not be read, so this plan cannot say what changes for them:`,
+          ...unreadable.map((u) => `  ${u.object} (${u.type}): ${u.reason}`),
+        ]
+      : [];
+  const trailer = [
+    ...refusedRead,
+    ...(diff.rebuilds.length > 0
       ? [
           "",
           `Refused: ${diff.rebuilds.length} change(s) need a rebuild, which ClickHouse cannot make to the existing table. ` +
             "A rebuild runs as its own migration (create the new table, backfill, verify, swap), not in place.",
           ...renderRebuildOps(diff.rebuildOps ?? []),
         ]
-      : [];
+      : []),
+  ];
   return renderChangeSet(diff, { ...(opts.title ? { title: opts.title } : {}), rules: CLASSIFIER_RULES, classes: CHANGE_CLASSES, trailer });
 }

package/src/clickhouse/plan/rules.ts

@@ -35,7 +35,7 @@ const DOCS = "https://clickhouse.com/docs/sql-reference/statements";
 const rule = (id: string, cls: ChangeClass, title: string, restriction: string, cite: string): ClassifierRule => classifierRule(id, cls, title, restriction, cite);
 
 export const CLASSIFIER_RULES = {
-  SQLCH200: rule("SQLCH200", "create", "Create an object", "A new database, table or view is created; nothing existing changes.", `${DOCS}/create`),
+  SQLCH200: rule("SQLCH200", "create", "Create an object", "A new database, table, view, dictionary, function, user, role or row policy is created, or a grantee gets its first grants; nothing existing changes.", `${DOCS}/create`),
   SQLCH201: rule(
     "SQLCH201",
     "metadata",
@@ -235,6 +235,49 @@ export const CLASSIFIER_RULES = {
     "ALTER TABLE ... MODIFY REFRESH changes the schedule of a refreshable materialized view.",
     `${DOCS}/alter/view`,
   ),
+  SQLCH245: rule(
+    "SQLCH245",
+    "metadata",
+    "Change a dictionary",
+    "A dictionary stores no data of its own: CREATE OR REPLACE DICTIONARY replaces its attributes, key, source, layout, lifetime or range, and it loads again from its source.",
+    `${DOCS}/create/dictionary`,
+  ),
+  SQLCH260: rule(
+    "SQLCH260",
+    "metadata",
+    "Change a function",
+    "A SQL user-defined function stores no data: CREATE OR REPLACE FUNCTION replaces its parameters and expression, and queries that call it use the new one.",
+    `${DOCS}/create/function`,
+  ),
+  SQLCH270: rule(
+    "SQLCH270",
+    "metadata",
+    "Change a role",
+    "ALTER ROLE ... SETTINGS replaces a role's settings; its grants are untouched.",
+    `${DOCS}/alter/role`,
+  ),
+  SQLCH271: rule(
+    "SQLCH271",
+    "metadata",
+    "Change a user",
+    "ALTER USER sets one clause (HOST, DEFAULT ROLE, DEFAULT DATABASE, GRANTEES, SETTINGS, VALID UNTIL, IDENTIFIED) and leaves the rest, the password and the grants untouched. Sessions already open keep the old settings.",
+    `${DOCS}/alter/user`,
+  ),
+  SQLCH272: rule(
+    "SQLCH272",
+    "metadata",
+    "Change a row policy",
+    "CREATE ROW POLICY OR REPLACE replaces the policy's condition, kind and roles; queries started after it see the new one.",
+    `${DOCS}/create/row-policy`,
+  ),
+  SQLCH273: rule("SQLCH273", "metadata", "Grant a privilege or a role", "GRANT adds a privilege or a role to a user or role; it takes effect for new queries.", `${DOCS}/grant`),
+  SQLCH274: rule(
+    "SQLCH274",
+    "metadata",
+    "Revoke a privilege or a role",
+    "REVOKE takes a privilege or a role away, including one granted by hand to a grantee the build declares grants for; queries that relied on it fail from then on.",
+    `${DOCS}/revoke`,
+  ),
   SQLCH250: rule("SQLCH250", "drop", "Drop an object", "DROP removes the object and, for a table, its data; it is not undone.", `${DOCS}/drop`),
 } as const satisfies Record<string, ClassifierRule>;
 

package/src/clickhouse/plan/schema.ts

@@ -4,15 +4,16 @@
  */
 
 import { readFileSync } from "node:fs";
-import { canonicalObject } from "./normalize";
-import type { SchemaObject } from "./diff";
+import { canonicalObject, grantsObject, liveCanonical, objectKey, scopeOf } from "./normalize";
+import { grantsByGrantee } from "../access";
+import type { SchemaObject, UnreadableEntry } from "./diff";
 import type { ClickHouseTarget } from "../live/bind";
-import { readLiveSchema } from "../live/catalog";
+import { readLiveAccess, readLiveSchema, type DeclaredAccess } from "../live/catalog";
 import { renderFor, type Topology } from "../topology";
 
 interface OutputDoc {
   dialect?: string;
-  objects?: Array<{ export: string; ddl: string }>;
+  objects?: Array<{ export: string; ddl: string; type?: string }>;
 }
 
 /** The objects a `chant build` output holds, keyed by export name; with a `topology`, each declaration as rendered for it (`../topology.ts`). */
@@ -21,7 +22,15 @@ export function schemaFromBuildOutput(json: string, defaultDatabase = "default",
   if (doc.dialect !== "clickhouse" || !Array.isArray(doc.objects)) {
     throw new Error("not a sql lexicon build output: expected { dialect: \"clickhouse\", objects: [...] }");
   }
-  return doc.objects.map((o) => ({ key: o.export, canonical: canonicalObject(renderFor(o.ddl, topology), defaultDatabase) }));
+  const out: SchemaObject[] = doc.objects
+    .filter((o) => o.type !== "ClickHouse::Grant")
+    .map((o) => ({ key: o.export, canonical: canonicalObject(renderFor(o.ddl, topology), defaultDatabase) }));
+  // Grants are compared per grantee (`../access.ts`).
+  for (const g of grantsByGrantee(doc.objects.map((o) => ({ ...o, type: o.type ?? "" })))) {
+    const canonical = grantsObject(g.ddl, g.grantee, defaultDatabase);
+    out.push({ key: objectKey(canonical), canonical });
+  }
+  return out;
 }
 
 export function schemaFromBuildFile(path: string, defaultDatabase = "default", topology?: Topology): SchemaObject[] {
@@ -34,7 +43,7 @@ export function schemaFromBuildFile(path: string, defaultDatabase = "default", t
  */
 export function keyedByQualifiedName(objects: readonly SchemaObject[]): Array<SchemaObject & { exportName: string }> {
   return objects.map((o) => ({
-    key: o.canonical.database ? `${o.canonical.database}.${o.canonical.name}` : o.canonical.name,
+    key: objectKey(o.canonical),
     canonical: o.canonical,
     exportName: o.key,
   }));
@@ -42,16 +51,30 @@ export function keyedByQualifiedName(objects: readonly SchemaObject[]): Array<Sc
 
 /**
  * What the server holds, keyed by `database.name`, limited to the databases
- * the declarations use (and the target's scope). The `default` database is
- * left out, as import leaves it out.
+ * the declarations use (and the target's scope) and the functions they
+ * declare (`scopeOf`). The `default` database is
+ * left out, as import leaves it out. An object whose definition chant cannot
+ * read is not left out silently (#3653): it goes in `unreadable`, when
+ * given, and otherwise fails the read.
  */
-export async function schemaFromServer(target: ClickHouseTarget, databases?: ReadonlySet<string>): Promise<SchemaObject[]> {
-  const live = await readLiveSchema(target);
-  return live
-    .filter((o) => !(o.type === "ClickHouse::Database" && o.name === "default"))
-    .filter((o) => !databases || databases.has(o.database ?? o.name))
-    .map((o) => ({
-      key: o.database ? `${o.database}.${o.name}` : o.name,
-      canonical: canonicalObject(o.statement, target.defaultDatabase),
-    }));
+export async function schemaFromServer(
+  target: ClickHouseTarget,
+  databases?: ReadonlySet<string>,
+  unreadable?: UnreadableEntry[],
+  access: readonly DeclaredAccess[] = [],
+): Promise<SchemaObject[]> {
+  const live = [...(await readLiveSchema(target)), ...(await readLiveAccess(target, access))];
+  const out: SchemaObject[] = [];
+  for (const o of live) {
+    if (o.type === "ClickHouse::Database" && o.name === "default") continue;
+    if (databases && !databases.has(scopeOf(o))) continue;
+    const key = objectKey(o);
+    try {
+      out.push({ key, canonical: liveCanonical(o, target.defaultDatabase) });
+    } catch (err) {
+      if (!unreadable) throw err;
+      unreadable.push({ object: key, type: o.type, reason: `its definition does not parse: ${(err as Error).message}` });
+    }
+  }
+  return out;
 }

package/src/clickhouse/plan/server-format.ts

@@ -9,7 +9,7 @@
 import { clickhouseQuery, type ClickHouseEndpoint } from "../http";
 import type { Change } from "./diff";
 
-const EXPRESSION_FIELDS = /^(orderBy|primaryKey|partitionBy|sampleBy|select|columns\.[^.]+\.(default|ttl)|indexes\..+|projections\..+)$/;
+const EXPRESSION_FIELDS = /^(orderBy|primaryKey|partitionBy|sampleBy|select|lambda|using|columns\.[^.]+\.(default|ttl)|indexes\..+|projections\..+)$/;
 
 const quote = (s: string) => `'${s.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
 

package/src/clickhouse/rebuild/backfill.ts

@@ -71,7 +71,7 @@ import { RebuildRefusal, type RebuildObservation } from "./observe";
 import { DDL_SETTINGS } from "./replicas";
 import { shardPartitions, sourcePartitionExpression, sourcePartitions } from "./partitions";
 import { rowHash } from "./verify";
-import { cutoverOf, observe, serverNow, syncReplica, utcLiteral, waitOn, type RebuildRun } from "./steps";
+import { cutoverOf, observe, serverCutoverProbe, syncReplica, utcLiteral, waitForCutover, waitOn, type CutoverProbe, type RebuildRun } from "./steps";
 import { intoShard, onShard, onShardServers, shardTable, SHARD_DELETE_SETTINGS, SHARD_INSERT_SETTINGS, type Shard } from "./shards";
 import { renderFor } from "../topology";
 
@@ -118,6 +118,8 @@ export interface BackfillDeps {
   receipts?: ClickHouseReceiptStore;
   /** Called after each partition is copied and its receipt written, with its shard on a cluster of shards; a test interrupts the backfill here. */
   afterPartition?: (partition: string, shard?: number) => void | Promise<void>;
+  /** What the wait for the cut-over reads. Default: the server's clock, queries and insert buffers. */
+  cutoverProbe?: CutoverProbe;
 }
 
 export async function backfill(run: RebuildRun, deps: BackfillDeps = {}): Promise<BackfillResult> {
@@ -128,16 +130,8 @@ export async function backfill(run: RebuildRun, deps: BackfillDeps = {}): Promis
   const n = o.names;
   const cutover = cutoverOf(run, o);
 
-  if (cutover !== undefined) {
-    // Rows before the cut-over are the backfill's; wait until the server's
-    // clock has passed it, so none of them is still to arrive.
-    for (;;) {
-      const now = await serverNow(run.target);
-      if (now > cutover) break;
-      run.log(`-- waiting ${Math.ceil((cutover - now) / 1000)}s for the cut-over at ${new Date(cutover).toISOString()}`);
-      await sleep(Math.min(cutover - now + 50, 10_000), run.signal);
-    }
-  }
+  // Rows before the cut-over are the backfill's; wait until none of them is still to arrive.
+  if (cutover !== undefined) await waitForCutover(run, cutover, deps.cutoverProbe ?? serverCutoverProbe(run, o), run.cutoverTimeoutMs !== undefined ? { timeoutMs: run.cutoverTimeoutMs } : {});
   await waitOn(run, n.database, n.name);
   await syncReplica(run, o, n.database, n.name);
 
@@ -335,17 +329,3 @@ async function copyAfterCutover(run: RebuildRun, o: RebuildObservation, later: L
   }
   return repaired;
 }
-
-function sleep(ms: number, signal?: AbortSignal): Promise<void> {
-  return new Promise((resolve, reject) => {
-    const t = setTimeout(resolve, ms);
-    signal?.addEventListener(
-      "abort",
-      () => {
-        clearTimeout(t);
-        reject(signal.reason);
-      },
-      { once: true },
-    );
-  });
-}

package/src/clickhouse/rebuild/observe.ts

@@ -57,11 +57,15 @@ export type DualWrite =
        * `cutoverColumn` is at or after the cut-over into the new table; the
        * backfill copies the rows before it, and the rows at or after it that
        * the table held before the view was made. The cut-over is the server's
-       * time when the view is made plus `cutoverDelay` (default `1m`), rounded
-       * up to a whole second, and the backfill starts once it has passed. The column must be a time
+       * time when the view is made plus `cutoverDelay` (default `5s`), rounded
+       * up to a whole second. The backfill starts once the server's clock has
+       * passed it and every write into the old table begun before it has
+       * finished (`cutoverTimeout` bounds that wait). The column must be a time
        * (`Date`, `DateTime`, `DateTime64`) that rows arrive in order of, give
        * or take the delay; a row that arrives later than that is caught by
-       * the verification, which then fails the run.
+       * the verification, which then fails the run. Writers that batch rows
+       * for longer than five seconds before inserting them need a delay
+       * longer than their batches.
        */
       mode: "materialized-view";
       cutoverColumn: string;

package/src/clickhouse/rebuild/op.ts

@@ -105,6 +105,13 @@ export interface ClickHouseRebuildOpConfig {
    * of the step waits this long. Default: `2m`.
    */
   replicaTimeout?: string;
+  /**
+   * Materialized-view mode: how long the backfill waits, once the cut-over
+   * has passed, for INSERTs into the old table begun before it to finish and
+   * for asynchronous inserts queued before it to be flushed, before it stops
+   * naming them. Default: `10m`.
+   */
+  cutoverTimeout?: string;
   /** Ownership stack. Default: `ownership.stack` in `chant.config.ts`. */
   stack?: string;
   /** Ownership env. Default: `ownership.env` in `chant.config.ts`. */
@@ -124,6 +131,7 @@ export interface ClickHouseRebuildArgs {
   retain?: string;
   mutationTimeout?: string;
   replicaTimeout?: string;
+  cutoverTimeout?: string;
   stack?: string;
   ownershipEnv?: string;
   cwd?: string;
@@ -158,6 +166,7 @@ export function ClickHouseRebuildOp(config: ClickHouseRebuildOpConfig): ClickHou
     ...(config.retain ? { retain: config.retain } : {}),
     ...(config.mutationTimeout ? { mutationTimeout: config.mutationTimeout } : {}),
     ...(config.replicaTimeout ? { replicaTimeout: config.replicaTimeout } : {}),
+    ...(config.cutoverTimeout ? { cutoverTimeout: config.cutoverTimeout } : {}),
     ...(config.stack ? { stack: config.stack } : {}),
     ...(config.ownershipEnv ? { ownershipEnv: config.ownershipEnv } : {}),
     ...(config.path && config.path !== "." ? { cwd: config.path } : {}),

package/src/clickhouse/rebuild/rebuild.e2e.test.ts

@@ -14,6 +14,9 @@
  *   backfill that fails part way leaves the new table and its receipts, and
  *   the next run skips the partitions already copied, verifies and swaps
  *   with no gate, and keeps the old table.
+ * - A small table (#3700): the backfill waits only the default cut-over
+ *   delay, and the wait sees an INSERT still running and an asynchronous
+ *   insert still buffered.
  * - An interrupted backfill: stopped after three partitions, then killed
  *   between a partition's INSERT and its receipt, then resumed: the receipts
  *   skip what was copied, the half-copied partition is cleared first, and the
@@ -38,6 +41,7 @@ import * as rebuildActivities from "../../op/activities/clickhouse-rebuild";
 import type { ClickHouseRebuildDeps } from "../../op/activities/clickhouse-rebuild";
 import { ClickHouseRebuildOp, type ClickHouseRebuildOpConfig } from "./op";
 import { clickhouseReceiptStore, RECEIPTS_DATABASE, RECEIPTS_TABLE, type ClickHouseReceiptStore } from "./receipts";
+import { serverCutoverProbe, type RebuildRun } from "./steps";
 
 const enabled = await dockerAvailable();
 let server: ScratchServer | undefined;
@@ -560,3 +564,61 @@ ORDER BY ${orderBy}`;
     await q("DROP TABLE shop.sessions__chant_new SYNC");
   }, 300_000);
 });
+
+describe.skipIf(!enabled)("a small table waits seconds for its cut-over (#3700)", () => {
+  const tinyDdl = (orderBy: string) => `CREATE TABLE shop.tiny (ts DateTime, id UInt64) ENGINE = MergeTree PARTITION BY toYYYYMM(ts) ORDER BY ${orderBy}`;
+  const V1: Obj = { export: "tiny", type: "ClickHouse::Table", dependsOn: ["shop"], ddl: tinyDdl("(ts, id)") };
+  const V2: Obj = { ...V1, ddl: tinyDdl("(id, ts)") };
+
+  test("thirty rows reach the swap gate with the default delay, a few seconds after the view is made", async () => {
+    await clickhouseApply({ buildPath: join(dir, writeBuild("tiny-v1.json", [DB, V1])), environment: "e2e" }, undefined, deps());
+    await q("INSERT INTO shop.tiny SELECT now() - toIntervalMinute(number), number FROM numbers(30)");
+    writeBuild("tiny-v2.json", [DB, V2]);
+    const lines: string[] = [];
+    const began = Date.now();
+    const r = await runOp(
+      {
+        name: "rebuild-tiny",
+        env: "e2e",
+        table: "shop.tiny",
+        dualWrite: { mode: "materialized-view", cutoverColumn: "ts" },
+        build: false,
+        path: dir,
+        output: "tiny-v2.json",
+        stack: MARKER.stack,
+        ownershipEnv: MARKER.env,
+      },
+      new Ledger().port,
+      { log: (l: string) => void lines.push(l) },
+    );
+    expect(r.status).toBe("gated");
+    expect(outcome(r, "VerifiedRows")).toBe(30);
+    // The delay is five seconds, rounded up to the next whole second.
+    const waited = lines.filter((l) => l.startsWith("-- waiting") && l.includes("for the cut-over"));
+    expect(waited.length).toBeGreaterThan(0);
+    expect(Number(/waiting (\d+)s/.exec(waited[0]!)![1])).toBeLessThanOrEqual(6);
+    expect(Date.now() - began).toBeLessThan(30_000);
+  }, 120_000);
+
+  test("the wait names an INSERT begun before the cut-over while it runs, and an asynchronous insert while it is buffered", async () => {
+    await q("CREATE TABLE shop.writes (ts DateTime, id UInt64) ENGINE = MergeTree ORDER BY ts");
+    const run = { target: { endpoint: server!.endpoint } } as unknown as RebuildRun;
+    const probe = serverCutoverProbe(run, { names: { database: "shop", name: "writes" }, replicated: false } as never);
+    const later = (await probe.now()) + 60_000;
+    expect(await probe.pendingWrites(later)).toEqual([]);
+
+    const slow = clickhouseQuery(server!.endpoint, "INSERT INTO shop.writes SELECT now(), number + sleepEachRow(1) FROM numbers(3) SETTINGS max_block_size = 1", { queryId: "e2e-3700-slow" });
+    await new Promise((r) => setTimeout(r, 500));
+    expect(await probe.pendingWrites(later)).toEqual(["e2e-3700-slow"]);
+    // Begun after the cut-over: not the backfill's to wait for.
+    expect(await probe.pendingWrites((await probe.now()) - 60_000)).toEqual([]);
+    await slow;
+    expect(await probe.pendingWrites(later)).toEqual([]);
+
+    await clickhouseQuery(server!.endpoint, "INSERT INTO shop.writes VALUES (now(), 7)", {
+      queryId: "e2e-3700-async",
+      settings: { async_insert: "1", wait_for_async_insert: "0", async_insert_busy_timeout_min_ms: "4000", async_insert_busy_timeout_max_ms: "4000", async_insert_use_adaptive_busy_timeout: "0" },
+    });
+    expect(await probe.pendingWrites(later)).toEqual(["e2e-3700-async"]);
+  }, 60_000);
+});