@intentius/chant-lexicon-sql 0.118.1 → 0.121.0

package/src/clickhouse/rebuild/verify.ts

@@ -24,6 +24,14 @@
  * later than the cut-over delay, with a time before the cut-over) stop the
  * swap too, rather than stay behind in the old table.
  *
+ * A table whose engine collapses rows that share a sorting key when parts
+ * merge (Summing, Replacing, Aggregating, Collapsing, VersionedCollapsing,
+ * Coalescing and Graphite, replicated or not), rebuilt into such an engine,
+ * is read under `FINAL` on both sides (#3674). The old table can hold unmerged parts with several rows for
+ * one key, which the copy inserts in one block and the new table collapses
+ * as it writes them, so the raw counts differ although the data is the same.
+ * Under `FINAL` both sides are compared as merged.
+ *
  * On a cluster of more than one shard (#3663) every shard is read, through
  * the profile's server, and each partition is compared per shard
  * (`shard<n>/<partition>`), so a shard the copy missed is a difference.
@@ -44,9 +52,11 @@ import { rebuildPlanSubject, RebuildRefusal, type RebuildObservation } from "./o
 import { sourcePartitionExpression } from "./partitions";
 import { cutoverOf, observe, syncReplica, utcLiteral, waitOn, type RebuildRun } from "./steps";
 import { shardTable } from "./shards";
+import { collapseSemantics, collapsesRows, splitTopLevel, type CollapseSemantics, type EngineColumn } from "./engines";
 
 export interface PartitionCheck {
   partition: string;
+  /** Rows; keys of the new sorting key when the tables were compared by key (#3727). */
   rows: number;
   checksum: string;
 }
@@ -67,23 +77,42 @@ export interface VerifyResult {
 
 /** A verification that found the tables different. */
 export class RebuildVerificationError extends Error {
+  /** The engine, when it collapses rows and the tables were compared under FINAL (#3674) or by sorting key (#3727). */
+  readonly collapsingEngine?: string;
   constructor(
     readonly table: string,
     readonly mismatches: Mismatch[],
+    /** How the tables were compared when an engine collapses rows; an engine's name alone means under FINAL. */
+    collapsed?: string | Collapsed,
   ) {
+    const how: Collapsed | undefined = typeof collapsed === "string" ? { engine: collapsed, how: "final" } : collapsed;
     const shown = mismatches
       .slice(0, 10)
-      .map((m) => `${m.partition}: old ${m.old ? `${m.old.rows} rows, checksum ${m.old.checksum}` : "none"}, new ${m.new ? `${m.new.rows} rows, checksum ${m.new.checksum}` : "none"}`)
+      .map((m) => `${m.partition}: old ${m.old ? `${m.old.rows} ${unit(how)}, checksum ${m.old.checksum}` : "none"}, new ${m.new ? `${m.new.rows} ${unit(how)}, checksum ${m.new.checksum}` : "none"}`)
       .join("; ");
+    const lateWrite =
+      `in materialized-view mode a row written to the old table later than the cut-over delay, with a time before the cut-over, ` +
+      `and in app mode a write the application did not stop`;
     super(
-      `${table}: the new table does not match the old one in ${mismatches.length} partition(s): ${shown}${mismatches.length > 10 ? "; ..." : ""}. ` +
-        `Nothing was swapped. In materialized-view mode a row written to the old table later than the cut-over delay, with a time before the cut-over, is the usual cause; ` +
-        `in app mode, a write the application did not stop.`,
+      `${table}: the new table does not match the old one in ${mismatches.length} partition(s): ${shown}${mismatches.length > 10 ? "; ..." : ""}. Nothing was swapped. ` +
+        (how?.how === "final"
+          ? `${table} is a ${how.engine}, which collapses rows that share a sorting key when parts merge, and both tables were compared under FINAL. ` +
+            `They still differ when the new table collapses the copied rows by its sorting key differently from how the old one merges them: ` +
+            `run OPTIMIZE TABLE ${table} FINAL, drop the new table if the Op kept it, and run the rebuild again, so the copy reads merged rows. ` +
+            `The other causes are ${lateWrite}.`
+          : how?.how === "keys"
+            ? `The new table is a ${how.engine}, which collapses rows that share a sorting key, so both tables were grouped by its sorting key and partition ` +
+              `and compared per key on ${how.describe ?? "the keys"}, which its merges keep. A key missing from the new table, or one whose values differ, is rows the copy lost or changed. ` +
+              `The usual cause is ${lateWrite}.`
+            : `The usual cause is ${lateWrite}.`),
     );
     this.name = "RebuildVerificationError";
+    if (how) this.collapsingEngine = how.engine;
   }
 }
 
+const unit = (how: Collapsed | undefined) => (how?.how === "keys" ? "keys" : "rows");
+
 /** How often a difference is read again before it counts, and how long apart. */
 const COMPARE_ATTEMPTS = 5;
 const COMPARE_INTERVAL_MS = 1000;
@@ -92,6 +121,81 @@ const COMPARE_INTERVAL_MS = 1000;
 interface SideCounts {
   all: Map<string, PartitionCheck>;
   before: Map<string, PartitionCheck>;
+  /** Grouped by sorting key (#3727): the rows read, of which `all` counts the keys. */
+  rows?: number;
+}
+
+/**
+ * How the tables were compared when an engine collapses rows: both under
+ * FINAL (#3674), or grouped by the new table's sorting key on what its
+ * engine's merges keep (#3727).
+ */
+export interface Collapsed {
+  engine: string;
+  how: "final" | "keys";
+  /** With `keys`: what is compared per key. */
+  describe?: string;
+}
+
+/**
+ * Both tables grouped by the new table's sorting key and its partition, each
+ * key compared on what the new engine's merges keep (#3727): its sums, its
+ * highest version, the sum of its signs. Each side is read raw, so the
+ * comparison is the same whichever parts have merged. The old side's columns
+ * are cast to the new table's types and named as there, so the sorting key,
+ * the partition key and the aggregates read the same on both sides. A
+ * partition is the new table's (`shard<n>/` before it on a cluster), and its
+ * rows are its keys.
+ */
+async function keyComparison(run: RebuildRun, o: RebuildObservation, columns: readonly EngineColumn[]): Promise<{ semantics: CollapseSemantics; read: (side: "old" | "new") => Promise<SideCounts> } | undefined> {
+  const n = o.names;
+  const [t] = await clickhouseQuery<{ engine_full: string; sorting_key: string; partition_key: string }>(
+    run.target.endpoint,
+    `SELECT engine_full, sorting_key, partition_key FROM system.tables WHERE database = ${sqlString(n.database)} AND name = ${sqlString(n.newName)}`,
+  );
+  if (!t || !o.newTable) return undefined;
+  const semantics = collapseSemantics(o.newTable.engine, t.engine_full, columns);
+  if (!semantics) return undefined;
+  // Every column of the sorting key must come from the old table, or the old rows have no key to group by.
+  const copied = new Set(o.copied.map((c) => c.name));
+  if (columns.some((c) => c.inSortingKey && !copied.has(c.name))) return undefined;
+
+  const types = new Map(columns.map((c) => [c.name, c.type]));
+  const sharding = run.sharding;
+  const cutover = cutoverOf(run, o);
+  const cut = run.dualWrite.mode === "materialized-view" ? (run.dualWrite as { cutoverColumn: string }).cutoverColumn : undefined;
+  const before = cut && copied.has(cut) && cutover !== undefined ? `${ident(cut)} < ${utcLiteral(cutover)}` : "1";
+  const key = splitTopLevel(t.sorting_key).filter(Boolean);
+  const parts = splitTopLevel(t.partition_key).filter(Boolean);
+  const partition = parts.length === 0 ? "'all'" : `toString(${parts.length === 1 ? parts[0] : `tuple(${parts.join(", ")})`})`;
+  const label = sharding ? `concat('shard', toString(__shard), '/', ${partition})` : partition;
+  const hash = `cityHash64(toString(tuple(${[...key, ...semantics.aggregates].join(", ")})))`;
+
+  const read = async (side: "old" | "new"): Promise<SideCounts> => {
+    const name = side === "old" ? n.name : n.newName;
+    const from = sharding ? shardTable(sharding, n.database, name) : side === "old" ? n.table : n.newTable;
+    const projected = o.copied.map((c) => (side === "old" ? `CAST(${ident(c.source)}, ${sqlString(types.get(c.name) ?? "String")}) AS ${ident(c.name)}` : ident(c.name)));
+    const rowsOf = `SELECT ${sharding ? "_shard_num AS __shard, " : ""}${projected.join(", ")} FROM ${from}`;
+    const perKey =
+      `SELECT ${label} AS __p, ${hash} AS __h, max(${before}) AS __b, count() AS __n, ${semantics.having ?? "1"} AS __k FROM (${rowsOf}) ` +
+      `GROUP BY __p${sharding ? ", __shard" : ""}${key.length > 0 ? `, ${key.join(", ")}` : ""}`;
+    // A key the engine removes (its sums all zero, its signs cancelled) is not compared, but its rows are counted as read.
+    const rows = await clickhouseQuery<{ p: string; rows: string | number; checksum: string; brows: string | number; bchecksum: string; raw: string | number }>(
+      run.target.endpoint,
+      `SELECT __p AS p, countIf(__k) AS rows, toString(sumIf(__h, __k)) AS checksum, countIf(__b AND __k) AS brows, toString(sumIf(__h, __b AND __k)) AS bchecksum, sum(__n) AS raw ` +
+        `FROM (${perKey}) GROUP BY p`,
+    );
+    const out: SideCounts = { all: new Map(), before: new Map(), rows: 0 };
+    for (const r of rows) {
+      out.rows! += Number(r.raw);
+      // A partition whose keys the engine all removes holds nothing in the new table, as if it had no rows.
+      if (Number(r.rows) === 0) continue;
+      out.all.set(r.p, { partition: r.p, rows: Number(r.rows), checksum: r.checksum });
+      if (Number(r.brows) > 0) out.before.set(r.p, { partition: r.p, rows: Number(r.brows), checksum: r.bchecksum });
+    }
+    return out;
+  };
+  return { semantics, read };
 }
 
 /**
@@ -103,21 +207,21 @@ async function compareTables(
   run: RebuildRun,
   o: RebuildObservation,
   opts: { attempts?: number; intervalMs?: number } = {},
-): Promise<{ old: SideCounts; new: SideCounts; mismatches: Mismatch[] }> {
+): Promise<{ old: SideCounts; new: SideCounts; mismatches: Mismatch[]; collapsed?: Collapsed }> {
   if (!o.newTable) throw new RebuildRefusal(`${o.names.key}: the new table is not there yet; the Create phase makes it`);
   const n = o.names;
   const cutover = cutoverOf(run, o);
   await waitOn(run, n.database, n.name);
   await waitOn(run, n.database, n.newName);
 
-  const types = new Map(
-    (
-      await clickhouseQuery<{ name: string; type: string }>(
-        run.target.endpoint,
-        `SELECT name, type FROM system.columns WHERE database = ${sqlString(n.database)} AND table = ${sqlString(n.newName)}`,
-      )
-    ).map((c) => [c.name, c.type]),
-  );
+  const newColumns = (
+    await clickhouseQuery<{ name: string; type: string; key: number | string; partition: number | string }>(
+      run.target.endpoint,
+      `SELECT name, type, is_in_sorting_key AS key, is_in_partition_key AS partition FROM system.columns ` +
+        `WHERE database = ${sqlString(n.database)} AND table = ${sqlString(n.newName)} ORDER BY position`,
+    )
+  ).map((c): EngineColumn => ({ name: c.name, type: c.type, inSortingKey: Number(c.key) === 1, inPartitionKey: Number(c.partition) === 1 }));
+  const types = new Map(newColumns.map((c) => [c.name, c.type]));
   const oldHash = rowHash(o.copied.map((c) => `CAST(${ident(c.source)}, ${sqlString(types.get(c.name) ?? "String")})`));
   const newHash = rowHash(o.copied.map((c) => ident(c.name)));
   const cut = run.dualWrite.mode === "materialized-view" ? o.copied.find((c) => c.name === (run.dualWrite as { cutoverColumn: string }).cutoverColumn) : undefined;
@@ -127,8 +231,15 @@ async function compareTables(
 
   // Across the shards of a cluster (#3663), every shard's rows, each partition counted per shard as `shard<n>/<partition>`.
   const sharding = run.sharding;
-  const read = async (table: string, name: string, partition: string, hash: string, before: string): Promise<SideCounts> => {
-    const from = sharding ? shardTable(sharding, n.database, name) : table;
+  // Tables whose engines collapse rows by sorting key are read as merged (#3674). Only when both do: a collapsing
+  // table rebuilt into a plain MergeTree keeps every row it copies, and is compared row for row.
+  const final = collapsesRows(o.live.engine) && collapsesRows(o.newTable.engine);
+  // A table that does not collapse rows rebuilt into one that does (#3727): the new table keeps one row per sorting
+  // key where the old one keeps them all, by design. Both are grouped by the new sorting key and compared on what
+  // the new engine's merges keep per key.
+  const keys = !final && collapsesRows(o.newTable.engine) ? await keyComparison(run, o, newColumns) : undefined;
+  const read = async (table: string, name: string, partition: string, hash: string, before: string, final: boolean): Promise<SideCounts> => {
+    const from = `${sharding ? shardTable(sharding, n.database, name) : table}${final ? " FINAL" : ""}`;
     const key = sharding ? `concat('shard', toString(_shard_num), '/', ${partition})` : partition;
     const rows = await clickhouseQuery<{ p: string; rows: string | number; checksum: string; brows: string | number; bchecksum: string }>(
       run.target.endpoint,
@@ -150,10 +261,18 @@ async function compareTables(
     await syncReplica(run, o, n.database, n.name);
     await syncReplica(run, o, n.database, n.newName);
     // The old table first: a row the view has yet to commit in the new one is read as missing there, never the other way round.
-    const before = await read(n.table, n.name, "_partition_id", oldHash, oldBefore);
-    const after = await read(n.newTable, n.newName, fromOld, newHash, newBefore);
+    const before = keys ? await keys.read("old") : await read(n.table, n.name, "_partition_id", oldHash, oldBefore, final);
+    const after = keys ? await keys.read("new") : await read(n.newTable, n.newName, fromOld, newHash, newBefore, final);
     const mismatches = differences(before.all, after.all);
-    if (mismatches.length === 0 || attempt >= attempts) return { old: before, new: after, mismatches };
+    if (mismatches.length === 0 || attempt >= attempts) {
+      return {
+        old: before,
+        new: after,
+        mismatches,
+        ...(final ? { collapsed: { engine: o.live.engine, how: "final" as const } } : {}),
+        ...(keys ? { collapsed: { engine: o.newTable.engine, how: "keys" as const, describe: keys.semantics.describe } } : {}),
+      };
+    }
     run.log(`-- ${n.key}: ${mismatches.length} partition(s) differ (${mismatches.map((m) => m.partition).slice(0, 5).join(", ")}); reading again (${attempt}/${attempts})`);
     await sleep(opts.intervalMs ?? COMPARE_INTERVAL_MS, run.signal);
   }
@@ -179,8 +298,8 @@ export const rowHash = (columns: readonly string[]): string => `cityHash64(toStr
  * {@link RebuildVerificationError} when it does not, so nothing is swapped.
  */
 export async function assertSameRows(run: RebuildRun, o: RebuildObservation): Promise<void> {
-  const { mismatches } = await compareTables(run, o);
-  if (mismatches.length > 0) throw new RebuildVerificationError(o.names.key, mismatches);
+  const { mismatches, collapsed } = await compareTables(run, o);
+  if (mismatches.length > 0) throw new RebuildVerificationError(o.names.key, mismatches, collapsed);
 }
 
 export async function verifyRebuild(run: RebuildRun): Promise<VerifyResult> {
@@ -188,15 +307,25 @@ export async function verifyRebuild(run: RebuildRun): Promise<VerifyResult> {
   if (o.state !== "rebuild") return { state: o.state, partitions: 0, rows: 0, verification: [], summary: `nothing to verify: ${o.state}` };
   const cutover = cutoverOf(run, o);
   const compared = await compareTables(run, o);
-  if (compared.mismatches.length > 0) throw new RebuildVerificationError(o.names.key, compared.mismatches);
+  if (compared.mismatches.length > 0) throw new RebuildVerificationError(o.names.key, compared.mismatches, compared.collapsed);
 
   const all = [...compared.old.all.values()];
-  const rows = all.reduce((s, v) => s + v.rows, 0);
+  const counted = all.reduce((s, v) => s + v.rows, 0);
+  const c = compared.collapsed;
+  // Grouped by sorting key (#3727), each partition counts keys; the old table's rows are still every row read.
+  const rows = c?.how === "keys" ? (compared.old.rows ?? counted) : counted;
   // What the approval binds: the rows before the cut-over (every row in app mode), which later writes do not change.
   const verification = [...compared.old.before.values()].sort((x, y) => x.partition.localeCompare(y.partition));
   const bound = verification.reduce((s, v) => s + v.rows, 0);
-  const range = cutover !== undefined ? ` (${bound} of them before the cut-over at ${new Date(cutover).toISOString()})` : "";
-  const summary = `${o.names.key}: ${all.length} partition(s), ${rows} row(s)${range}, counts and checksums equal in the old and new tables`;
+  const range = cutover !== undefined ? ` (${bound} ${c?.how === "keys" ? "key(s)" : "of them"} before the cut-over at ${new Date(cutover).toISOString()})` : "";
+  const counts = c?.how === "keys" ? `${rows} row(s) of the old table as ${counted} key(s)` : `${rows} row(s)`;
+  const merged =
+    c?.how === "final"
+      ? `, read under FINAL (${c.engine})`
+      : c?.how === "keys"
+        ? `, grouped by the sorting key of the new ${c.engine} and compared on ${c.describe}`
+        : "";
+  const summary = `${o.names.key}: ${all.length} partition(s), ${counts}${range}, counts and checksums equal in the old and new tables${merged}`;
   run.log(`-- ${summary}`);
   return {
     state: o.state,

package/src/clickhouse/testing/fake-server.ts

@@ -24,6 +24,8 @@ export interface FakeOptions {
   /** Answer every query with this status and body (an auth failure, say). */
   fail?: { status: number; body: string };
   version?: string;
+  /** The SQL user-defined functions `system.functions` lists. Default: none. */
+  functions?: Array<{ name: string; statement: string }>;
 }
 
 const row = (o: Record<string, unknown>) => `${JSON.stringify(o)}\n`;
@@ -52,8 +54,7 @@ export async function fakeClickHouse(objects: FakeObject[], options: FakeOptions
           .join(""),
       );
     }
-    // No SQL user-defined functions.
-    if (/FROM system\.functions/.test(body)) return ok("");
+    if (/FROM system\.functions/.test(body)) return ok((options.functions ?? []).map((f) => row(f)).join(""));
     const show = /^SHOW CREATE (DATABASE|TABLE) (`(?:[^`]|``)*`)(?:\.(`(?:[^`]|``)*`))?$/.exec(body.trim());
     if (show) {
       const [db, name] = show[3] ? [unquote(show[2]!), unquote(show[3])] : [undefined, unquote(show[2]!)];

package/src/codegen/classifier-page.ts

@@ -70,7 +70,7 @@ ALTER TABLE \`analytics\`.\`events\` MODIFY COLUMN \`kind\` LowCardinality(Strin
 --   export const { op } = ClickHouseRebuildOp({ name: "rebuild-analytics-sessions", env: "<env>", table: "analytics.sessions", dualWrite: { mode: "materialized-view", cutoverColumn: "ts" } });
 \`\`\`
 
-A rebuild is never DDL: it is a step naming \`ClickHouseRebuildOp\`, or for a view or a database, which the Op does not rebuild, a manual step. A column or object drop is a statement marked destructive. Each \`CREATE\` carries the project's ownership marker, as the applier stamps it. The command exits 2 when a step is not a statement. Two things the applier asks the server are left out: whether a difference is formatting only, and whether a database is empty before it is dropped.
+A rebuild is never DDL: it is a step naming \`ClickHouseRebuildOp\`, or for a view or a database, which the Op does not rebuild, a manual step. A column or object drop is a statement marked destructive. Each \`CREATE\` carries the project's ownership marker, as the applier stamps it. The command exits 2 when a step is not a statement. A migration runner's history table (\`schema_migrations\`, \`goose_db_version\`; see [Importing a Live Server](../importing/)) is never dropped: the report names it in a hint. Two things the applier asks the server are left out: whether a difference is formatting only, and whether a database is empty before it is dropped.
 
 \`--topology\` renders the statements for where they will run: \`single\` (the default), \`cluster:<name>\`, \`replicated\` or \`cloud\`. On a cluster every statement carries \`ON CLUSTER\` and a MergeTree-family table becomes \`Replicated*MergeTree\` with its Keeper path; in a \`Replicated\` database the tables are \`Replicated*MergeTree\` with no path and no \`ON CLUSTER\`. \`diffStatements(before, after, { topology })\` takes the same choice; see [Applying to a Server](../applying/#topology).
 

package/src/config.ts

@@ -104,6 +104,16 @@ export const sqlProfileSchema = z.strictObject({
   databases: z.array(z.string()).optional(),
   /** The database an unqualified declaration is created in. `default` when omitted. */
   defaultDatabase: z.string().optional(),
+  /**
+   * ClickHouse import (#3718): the SQL user-defined functions `chant import
+   * --from <env>` adopts besides those the imported objects use, by name, or
+   * by prefix with a trailing `*` (`shop_*`). A function belongs to no
+   * database, so without this only the functions the imported tables, views
+   * and dictionaries use (and the functions those call) are imported. A use
+   * is found by name or, since ClickHouse stores a call as the function's
+   * body, by that body (#3745).
+   */
+  importFunctions: z.array(z.string()).optional(),
   /**
    * ClickHouse: the topology this environment runs, which the applier, the
    * plan and the rebuild migration render every statement for: `single`,
@@ -140,8 +150,11 @@ export const sqlProfileSchema = z.strictObject({
   /**
    * Postgres apply: how long one statement waits for its lock before it
    * fails (`lock_timeout`), in milliseconds, so a change blocked behind a
-   * long query fails fast instead of queueing every query behind it. 5000
-   * when omitted.
+   * long query fails fast instead of queueing every query behind it. The
+   * catalog reads (`chant lifecycle diff --live`, `chant sql plan`, the
+   * import, and an apply's read before its first statement) wait under it
+   * too, and name the session holding the lock when it passes. 5000 when
+   * omitted.
    */
   lockTimeoutMs: z.number().int().nonnegative().optional(),
   /**
@@ -156,11 +169,12 @@ export const sqlProfileSchema = z.strictObject({
    */
   scanTimeoutMs: z.number().int().nonnegative().optional(),
   /**
-   * Postgres: whether chant manages access in this environment (#3681): the
+   * Whether chant manages access in this environment. Postgres (#3681): the
    * declared row-level security policies and each table's row-level security,
    * the declared roles (created when missing, never dropped), and the
    * privileges on the declared objects and the default privileges, which a
-   * plan compares and an apply grants and revokes. Off when omitted: the
+   * plan compares and an apply grants and revokes. ClickHouse (#3716): the
+   * declared users, roles, row policies and grants. Off when omitted: the
    * access declarations are not planned or applied, and the server's access
    * is not read, so a project that manages access elsewhere is not affected.
    */

package/src/core/foreign-tables.ts

@@ -0,0 +1,28 @@
+/**
+ * Tables another tool keeps its migration history in, by name, with the tool.
+ *
+ * Both dialects' catalog readers mark a table by one of these names as
+ * foreign: import leaves it out with a warning, and a plan names it in a hint
+ * instead of proposing a drop, since chant changing it would change what that
+ * tool owns (#3047 question 10, #3676). golang-migrate, dbmate, goose and
+ * Flyway keep the same tables on ClickHouse as on Postgres.
+ */
+export const FOREIGN_TABLES: Readonly<Record<string, string>> = {
+  _prisma_migrations: "Prisma Migrate",
+  schema_migrations: "a migration runner (Rails, golang-migrate, dbmate)",
+  ar_internal_metadata: "Rails",
+  django_migrations: "Django",
+  alembic_version: "Alembic",
+  __drizzle_migrations: "drizzle-kit",
+  flyway_schema_history: "Flyway",
+  goose_db_version: "goose",
+  knex_migrations: "Knex",
+  knex_migrations_lock: "Knex",
+  SequelizeMeta: "Sequelize",
+  pgmigrations: "node-pg-migrate",
+};
+
+/** The tool that keeps a table by this name, or undefined when none does. */
+export function foreignTool(name: string): string | undefined {
+  return Object.prototype.hasOwnProperty.call(FOREIGN_TABLES, name) ? FOREIGN_TABLES[name] : undefined;
+}

package/src/files/clickhouse.ts

@@ -1,23 +1,79 @@
 /**
  * A file of ClickHouse DDL as the objects it declares: one per `CREATE
  * DATABASE`, `TABLE`, `VIEW`, `MATERIALIZED VIEW`, `DICTIONARY` or `FUNCTION`, as `chant import
- * <file>.sql` reads them. Any other statement, and any statement the parser
- * refuses, is a problem naming it.
+ * <file>.sql` reads them, and one per `CREATE USER`, `CREATE ROLE`, `CREATE
+ * ROW POLICY` and `GRANT` (#3711), each read by its tag, so what the tag
+ * refuses (`IDENTIFIED BY`, a `REVOKE`) is refused with the tag's message.
+ * Any other statement, and any statement the parser refuses, is a problem
+ * naming it.
  *
  * With `schema` (a database), an object's unqualified name is qualified with
- * it; a function belongs to no database and keeps its name.
+ * it, and so is the table a row policy or a grant is on; a function, a user
+ * and a role belong to no database and keep their names.
  */
 
 import type { Declarable } from "@intentius/chant/declarable";
 import { isTrivia, tokenizeText } from "../clickhouse/tokens";
 import { parseCreate } from "../clickhouse/parser";
-import { database, dictionary, func, table, view } from "../clickhouse/entities";
+import { CLICKHOUSE_ENTITY_TYPES, database, dictionary, func, grant, policy, role, table, user, view } from "../clickhouse/entities";
+import { isAccessStatement, parseAccess, type AccessNode } from "../clickhouse/access";
+import type { Token } from "../clickhouse/tokens";
 import { CLICKHOUSE_TAG_OF } from "../clickhouse/import/generator";
 import { describeStatement, type ImportedObject } from "../clickhouse/import/ir";
 import { firstLine, quoteBare, splitTokens, tokensText, type ReadOptions } from "./common";
+import { templateSafe, templateStrings } from "./postgres";
 
 type Tag = (strings: TemplateStringsArray, ...values: unknown[]) => Declarable;
-const TAGS: Record<string, Tag> = { database, table, view, dictionary, func };
+const TAGS: Record<string, Tag> = { database, table, view, dictionary, func, user, role, policy, grant };
+
+/** Tokens with `[db.]name`'s bare name qualified with `schema`. A `*` (all tables) is left alone. */
+function qualifyBare(tokens: Token[], span: { from: number; to: number } | undefined, schema: string): string | undefined {
+  if (!span) return undefined;
+  const name = tokens.slice(span.from, span.to).filter((t) => !isTrivia(t));
+  const [only] = name;
+  if (!only || name.length !== 1 || (only.kind !== "ident" && only.kind !== "qident")) return undefined;
+  const at = tokens.indexOf(only);
+  return tokensText(tokens.slice(0, at)) + `${quoteBare(schema)}.` + tokensText(tokens.slice(at));
+}
+
+/** An access statement as the object its tag declares (#3711), or the tag's refusal. */
+function readAccess(sql: string, options: ReadOptions): ImportedObject | string {
+  let tokens = tokenizeText(sql, 0);
+  let node: AccessNode;
+  try {
+    node = parseAccess(tokens);
+  } catch (e) {
+    // What the tag refuses (a password) is refused in the tag's words.
+    const words = sql.replace(/--[^\n]*\n/g, " ").trim().split(/\s+/, 2).map((w) => w.toUpperCase());
+    const tag = words[0] === "CREATE" ? (words[1] === "USER" ? user : words[1] === "ROLE" ? role : policy) : grant;
+    try {
+      tag(templateStrings([templateSafe(sql)]));
+    } catch (refused) {
+      return refused instanceof Error ? refused.message : String(refused);
+    }
+    throw e;
+  }
+  if (options.schema) {
+    const qualified = qualifyBare(tokens, node.statement === "rowPolicy" ? node.table : node.statement === "grant" ? node.target : undefined, options.schema);
+    if (qualified !== undefined) {
+      sql = qualified;
+      tokens = tokenizeText(sql, 0);
+      node = parseAccess(tokens);
+    }
+  }
+  const tag = node.statement === "user" ? user : node.statement === "role" ? role : node.statement === "rowPolicy" ? policy : grant;
+  let props: { name: string; database?: string; table?: string };
+  try {
+    props = (tag(templateStrings([templateSafe(sql)])) as unknown as { props: typeof props }).props;
+  } catch (e) {
+    return e instanceof Error ? e.message : String(e);
+  }
+  if (node.statement === "user") return { type: CLICKHOUSE_ENTITY_TYPES.user, name: props.name, ddl: sql };
+  if (node.statement === "role") return { type: CLICKHOUSE_ENTITY_TYPES.role, name: props.name, ddl: sql };
+  if (node.statement === "rowPolicy") return { type: CLICKHOUSE_ENTITY_TYPES.rowPolicy, ...(props.database ? { database: props.database } : {}), name: props.name, ddl: sql };
+  // A grant has no name of its own: what it grants, on what, to whom.
+  return { type: CLICKHOUSE_ENTITY_TYPES.grant, name: `grant ${tokensText(tokens.slice(tokens.findIndex((t) => !isTrivia(t)) + 1)).replace(/\s+/g, " ").trim().toLowerCase()}`, ddl: sql };
+}
 
 /** The tag that declares an entity type. */
 export function clickhouseTag(type: string): Tag | undefined {
@@ -30,6 +86,16 @@ export function readClickHouse(ddl: string, options: ReadOptions, problems: stri
   const objects: ImportedObject[] = [];
   for (const stmt of splitTokens(tokenizeText(ddl, 0), isTrivia)) {
     let sql = tokensText(stmt);
+    if (isAccessStatement(sql)) {
+      try {
+        const read = readAccess(sql, options);
+        if (typeof read === "string") problems.push(`${read}: ${firstLine(sql)}`);
+        else objects.push(read);
+      } catch (e) {
+        problems.push(`does not parse (${e instanceof Error ? e.message : String(e)}): ${firstLine(sql)}`);
+      }
+      continue;
+    }
     if (!/^CREATE\b/i.test(sql)) {
       problems.push(`not a CREATE statement: ${firstLine(sql)}`);
       continue;

package/src/files/postgres.ts

@@ -45,7 +45,8 @@ const isName = (t: Token | undefined): boolean => t !== undefined && (t.kind ===
 const word = (t: Token | undefined, ...words: string[]): boolean => t !== undefined && t.kind === "ident" && words.includes(t.text.toUpperCase());
 
 /** Text that sits inside a template literal: a backquote and `${` escaped. */
-const templateSafe = (s: string) => s.replace(/\\(?=`|\$\{)/g, "\\\\").replace(/`/g, "\\`").replace(/\$\{/g, "\\${");
+/** Text that sits inside a template literal: a backquote and `${` escaped. */
+export const templateSafe = (s: string) => s.replace(/\\(?=`|\$\{)/g, "\\\\").replace(/`/g, "\\`").replace(/\$\{/g, "\\${");
 
 interface PgFileObject {
   node: StatementNode;

package/src/import-parser.ts

@@ -25,6 +25,8 @@ const CLICKHOUSE_ONLY = [
   /^\s*CREATE\s+(?:DATABASE|(?:OR\s+REPLACE\s+)?DICTIONARY)\b/im,
   // A SQL function: CREATE FUNCTION f AS (x) -> ...
   /^\s*CREATE\s+(?:OR\s+REPLACE\s+)?FUNCTION\s+[^\s(]+(?:\s+ON\s+CLUSTER\s+\S+)?\s+AS\s*\(/im,
+  // A row policy (#3711): Postgres says CREATE POLICY.
+  /^\s*CREATE\s+(?:OR\s+REPLACE\s+)?ROW\s+POLICY\b/im,
 ];
 const POSTGRES_ONLY = [
   /^\s*CREATE\s+(?:OR\s+REPLACE\s+)?(?:SCHEMA|EXTENSION|TYPE|DOMAIN|SEQUENCE|(?:UNIQUE\s+)?INDEX|(?:CONSTRAINT\s+)?TRIGGER|POLICY|ROLE|FUNCTION|PROCEDURE|(?:UNLOGGED|TEMP|TEMPORARY)\s+TABLE)\b/im,

package/src/migration-statements.test.ts

@@ -111,6 +111,39 @@ describe("ClickHouse", () => {
     expect((d.steps[0] as { detail: string }).detail).toMatch(/drop and create this object instead/);
   });
 
+  test("a user the environment creates gets its declared clauses with ALTER USER, its password left alone (#3717)", () => {
+    const reader: Obj = { export: "reader", type: "ClickHouse::Role", ddl: "CREATE ROLE reader" };
+    const app: Obj = {
+      export: "app",
+      type: "ClickHouse::User",
+      dependsOn: ["reader", "analytics"],
+      ddl: "CREATE USER app HOST LOCAL DEFAULT ROLE reader DEFAULT DATABASE analytics SETTINGS max_memory_usage = 1000000000",
+    };
+    const readerToApp: Obj = { export: "readerToApp", type: "ClickHouse::Grant", dependsOn: ["reader", "app"], ddl: "GRANT reader TO app" };
+    const d = diffStatements(build("clickhouse", [DB]), build("clickhouse", [DB, reader, app, readerToApp]));
+    const sql = sqlOf(d);
+    expect(sql).toEqual([
+      "CREATE ROLE reader",
+      "ALTER USER `app` HOST LOCAL",
+      "ALTER USER `app` DEFAULT DATABASE analytics",
+      "ALTER USER `app` SETTINGS max_memory_usage = 1000000000",
+      "GRANT `reader` TO `app`",
+      // A default role must be held first.
+      "ALTER USER `app` DEFAULT ROLE reader",
+    ]);
+    expect(sql.join("\n")).not.toMatch(/CREATE USER|IDENTIFIED/);
+    expect(d.steps.filter((s) => s.object === "app").map((s) => (s as StatementStep).rule)).toEqual(["SQLCH271", "SQLCH271", "SQLCH271"]);
+
+    // Without grants to it, the default role is set with the rest.
+    const alone = sqlOf(diffStatements(build("clickhouse", [DB, reader]), build("clickhouse", [DB, reader, app])));
+    expect(alone).toEqual([
+      "ALTER USER `app` HOST LOCAL",
+      "ALTER USER `app` DEFAULT DATABASE analytics",
+      "ALTER USER `app` SETTINGS max_memory_usage = 1000000000",
+      "ALTER USER `app` DEFAULT ROLE reader",
+    ]);
+  });
+
   test("two identical builds take no steps", () => {
     expect(diffStatements(CH_BEFORE, CH_BEFORE)).toEqual({ dialect: "clickhouse", defaultDatabase: "default", topology: "single", steps: [], refused: false, hints: [] });
   });

package/src/migration-statements.ts

@@ -112,6 +112,8 @@ function clickhouseStatements(beforeJson: string, afterJson: string, options: Di
     current: new Map(before.map((o) => [o.exportName, o.canonical])),
     keyOf: (o) => o.exportName,
     allowDestructive: true,
+    // A user whose password is the environment's exists by the time a migration runs: the GRANTs to it need it too (#3717).
+    environmentCreatesUsers: true,
     topology,
     ...(options.marker ? { marker: options.marker } : {}),
   });

package/src/op/__snapshots__/watch-pipeline.test.ts.snap

@@ -62,7 +62,7 @@ jobs:
 `;
 
 exports[`generateOpsPipeline renders the sql drift watch (#3642) > GitLab: a schedule-only job with the reader's credentials and no token of its own 1`] = `
-"# chant Ops (#927, #2084, #2293) — one job per Op under stage "ops", each
+"# chant Ops — one job per Op under stage "ops", each
 # selected by its own rules:. A merge_request_event or push job needs no
 # setup; its rule fires on the event itself.
 #

package/src/op/activities/clickhouse-emulator.ts

@@ -10,9 +10,28 @@
  * `upstreamPin` already tracks the LTS tags (`-lts`).
  */
 
-import { emulatorLifecycle, type EmulatorCapability, type EmulatorSpec } from "@intentius/chant/op";
+import { emulatorLifecycle, type EmulatorCapability, type EmulatorIdentity, type EmulatorSpec } from "@intentius/chant/op";
 import { clickhouseImage } from "../../spec/pin";
 
+/**
+ * The server's UUID (#3673), read in the container and through the endpoint
+ * `status` prints, so a server of another origin on the same port is told
+ * apart from the emulator.
+ */
+export const CLICKHOUSE_EMULATOR_IDENTITY: EmulatorIdentity = {
+  server: "ClickHouse",
+  command: ["clickhouse-client", "-q", "SELECT serverUUID()"],
+  async probe(endpoint) {
+    const url = new URL(endpoint);
+    url.searchParams.set("query", "SELECT serverUUID(), version() FORMAT TSV");
+    const res = await fetch(url, { signal: AbortSignal.timeout(5_000) });
+    const body = (await res.text()).trim();
+    if (!res.ok) throw new Error(`HTTP ${res.status}: ${body.split("\n")[0]}`);
+    const [id = "", version = "(unknown version)"] = body.split("\t");
+    return { id, label: `ClickHouse ${version}` };
+  },
+};
+
 export const CLICKHOUSE_EMULATOR_SPEC: EmulatorSpec = {
   name: "chant-clickhouse",
   image: clickhouseImage(),
@@ -21,6 +40,8 @@ export const CLICKHOUSE_EMULATOR_SPEC: EmulatorSpec = {
   // The image disables network access for the passwordless default user
   // unless this is set; a local server is reached as `default` with no password.
   runArgs: ["-e", "CLICKHOUSE_SKIP_USER_SETUP=1"],
+  identity: CLICKHOUSE_EMULATOR_IDENTITY,
+  credentials: "user default, no password",
 };
 
 /** `CLICKHOUSE_URL` is the variable binding reads when no `sql.profiles.<env>` names a server. */