@intentius/chant-lexicon-sql 0.124.48 → 0.124.49

package/src/clickhouse/plan/statements.ts

@@ -0,0 +1,306 @@
+/**
+ * The ClickHouse statement classifier (#4407): a migration file's statements,
+ * each given the class and rule the schema-diff classifier (`./rules.ts`)
+ * gives the same change. It reads text and contacts no server.
+ *
+ * `CREATE` goes through the declaration parser (`../parser.ts`), which also
+ * tells the classifier a table's keys when the file creates the table. The
+ * parser has no `ALTER` grammar, so `ALTER`, `DROP` and `RENAME` are read word
+ * by word from the same tokens.
+ *
+ * A statement alone cannot decide some changes, and those answer `undecided`
+ * naming both rules, the way `./disruption.ts` answers `unknown`:
+ *
+ * - `MODIFY COLUMN c <type>` is SQLCH210 (a rewrite), or SQLCH211 (a rebuild)
+ *   when `c` is in a key. The file states the keys only when it creates the
+ *   table.
+ * - `DROP COLUMN` is SQLCH202 or SQLCH213, and `RENAME COLUMN` SQLCH212 or
+ *   SQLCH213, on the same condition.
+ * - `MODIFY ORDER BY` that the same `ALTER` pairs with `ADD COLUMN` is SQLCH216
+ *   when it only appends those columns and SQLCH220 (a rebuild) otherwise.
+ *   Without such a column it is SQLCH220.
+ */
+
+import { MERGE_TREE_SETTINGS } from "../../generated/clickhouse";
+import { isTrivia, tokenizeText, type Token } from "../tokens";
+import { parseCreate, type CreateNode } from "../parser";
+import type { Span } from "../../core/cursor";
+import { classifiedOf, splitTopLevel, sqlHead, type StatementClassified, type StatementVerdict } from "../../core/statement-class";
+import { CHANGE_CLASSES, CLASSIFIER_RULES, type ClassifierRuleId } from "./rules";
+
+const classified = (id: ClassifierRuleId, extra: Partial<StatementClassified> = {}): StatementClassified => classifiedOf(CLASSIFIER_RULES, CHANGE_CLASSES.label, id, extra);
+
+type Reading = { rules: Array<{ id: ClassifierRuleId; extra?: Partial<StatementClassified> }> } | { undecided: ClassifierRuleId[]; note: string } | { none: string } | { data: string };
+
+const rule = (id: ClassifierRuleId, extra?: Partial<StatementClassified>): Reading => ({ rules: [{ id, ...(extra ? { extra } : {}) }] });
+
+/** Settings only a stream engine takes (the same list `./disruption.ts` reads). */
+const STREAM_SETTING = /^(?:kafka_|rabbitmq_|nats_|s3queue_|azure_queue_|keeper_path$)/;
+
+/** What a table the file creates tells later statements in it. */
+interface Created {
+  /** The columns in the sorting, primary, partition or sampling key. */
+  keys: Set<string>;
+  orderBy?: string;
+}
+
+const bare = (name: string): string => (name.split(".").pop() ?? name).replace(/^[`"]|[`"]$/g, "").toLowerCase();
+
+const text = (tokens: readonly Token[], s: Span | undefined): string => (s ? tokens.slice(s.from, s.to).filter((t) => !isTrivia(t)).map((t) => t.text).join(" ") : "");
+
+/** The identifier tokens of a key expression that are not function names. */
+function columnsOf(tokens: readonly Token[], s: Span | undefined): string[] {
+  if (!s) return [];
+  const toks = tokens.slice(s.from, s.to).filter((t) => !isTrivia(t));
+  return toks.flatMap((t, i) => ((t.kind === "ident" || t.kind === "qident") && toks[i + 1]?.text !== "(" ? [bare(t.text)] : []));
+}
+
+function nameAt(words: readonly string[], i: number): { name: string; next: number } {
+  let name = words[i] ?? "";
+  let j = i + 1;
+  while (words[j] === "." && words[j + 1] !== undefined) {
+    name += `.${words[j + 1]}`;
+    j += 2;
+  }
+  return { name, next: j };
+}
+
+const KNOWN_VERBS = new Set(["ADD", "DROP", "RENAME", "MODIFY", "COMMENT", "CLEAR", "MATERIALIZE", "UPDATE", "DELETE", "ATTACH", "DETACH", "FREEZE", "FETCH", "MOVE", "REPLACE", "RESET", "REMOVE", "APPLY", "UNFREEZE", "EXECUTE"]);
+
+/** Normalise a key expression for comparing an old and a new one: no spaces, no outer parentheses. */
+const keyText = (words: readonly string[]): string => {
+  let t = words.join("").toLowerCase();
+  while (t.startsWith("(") && t.endsWith(")")) t = t.slice(1, -1);
+  return t;
+};
+
+interface TableContext {
+  created?: Created;
+  /** Columns the same ALTER adds. */
+  added: Set<string>;
+}
+
+function columnVerdict(kind: "drop" | "rename", col: string, ctx: TableContext): Reading {
+  const plain: ClassifierRuleId = kind === "drop" ? "SQLCH202" : "SQLCH212";
+  const extra = kind === "drop" ? { destructive: true } : undefined;
+  if (ctx.created) {
+    return ctx.created.keys.has(bare(col)) ? rule("SQLCH213", extra) : rule(plain, extra);
+  }
+  return { undecided: [plain, "SQLCH213"], note: `${col} is a column change or, in a key, a rebuild; the file does not create the table, so its keys are not known offline` };
+}
+
+function tableAction(a: readonly string[], ctx: TableContext): Reading {
+  const u = a.map((w) => w.toUpperCase());
+  const first = u[0];
+  const second = u[1];
+  if (first === "ADD") {
+    if (second === "COLUMN") return rule("SQLCH201");
+    if (second === "INDEX") return rule("SQLCH204");
+    if (second === "PROJECTION") return rule("SQLCH214");
+    if (second === "CONSTRAINT") return rule("SQLCH215");
+    return { none: `ALTER TABLE ${u.slice(0, 2).join(" ")}` };
+  }
+  if (first === "DROP") {
+    if (second === "COLUMN") {
+      const k = u[2] === "IF" ? 4 : 2;
+      return columnVerdict("drop", a[k] ?? "column", ctx);
+    }
+    if (second === "INDEX") return rule("SQLCH204");
+    if (second === "PROJECTION") return rule("SQLCH214");
+    if (second === "CONSTRAINT") return rule("SQLCH215");
+    return { none: `ALTER TABLE ${u.slice(0, 2).join(" ")} (removes data parts, no rule covers it)` };
+  }
+  if (first === "CLEAR") {
+    if (second === "INDEX") return rule("SQLCH204");
+    if (second === "PROJECTION") return rule("SQLCH214");
+    return { none: `ALTER TABLE CLEAR ${second ?? ""}`.trim() };
+  }
+  if (first === "RENAME" && second === "COLUMN") {
+    const k = u[2] === "IF" ? 4 : 2;
+    return columnVerdict("rename", a[k] ?? "column", ctx);
+  }
+  if (first === "COMMENT" && second === "COLUMN") return rule("SQLCH203");
+  if (first === "MODIFY" || first === "RESET") {
+    if (second === "COLUMN") {
+      let k = 2;
+      if (u[k] === "IF" && u[k + 1] === "EXISTS") k += 2;
+      const col = a[k] ?? "column";
+      const after = u[k + 1];
+      if (after === "REMOVE") {
+        const what = u[k + 2];
+        return what === "CODEC" ? rule("SQLCH208") : what === "COMMENT" ? rule("SQLCH203") : what === "TTL" ? rule("SQLCH205") : rule("SQLCH207");
+      }
+      if (after === "DEFAULT" || after === "MATERIALIZED" || after === "ALIAS" || after === "EPHEMERAL") return rule("SQLCH207");
+      if (after === "CODEC") return rule("SQLCH208");
+      if (after === "COMMENT") return rule("SQLCH203");
+      if (after === "TTL") return rule("SQLCH205");
+      if (after === "MODIFY" || after === "RESET") return { none: "MODIFY COLUMN ... SETTING" };
+      if (after === undefined) return { none: "MODIFY COLUMN with no change" };
+      if (ctx.created) return ctx.created.keys.has(bare(col)) ? rule("SQLCH211") : rule("SQLCH210");
+      return { undecided: ["SQLCH210", "SQLCH211"], note: `a type change of ${col} is a background rewrite or, when ${col} is in a key, a rebuild; the file does not create the table, so its keys are not known offline` };
+    }
+    if (second === "ORDER" && u[2] === "BY") {
+      const next = a.slice(3);
+      if (ctx.created?.orderBy !== undefined) {
+        const was = ctx.created.orderBy;
+        const now = keyText(next);
+        const tail = now.startsWith(was) ? now.slice(was.length) : undefined;
+        const extra = tail === undefined ? [] : tail.split(/[^a-z0-9_]+/).filter(Boolean);
+        if (tail !== undefined && extra.length > 0 && extra.every((c) => ctx.added.has(c))) return rule("SQLCH216");
+        return rule("SQLCH220");
+      }
+      const mentions = next.some((w) => ctx.added.has(bare(w)));
+      if (mentions) return { undecided: ["SQLCH216", "SQLCH220"], note: "the new key appends the columns this ALTER adds only if its other elements are the old key's, which the file does not state" };
+      return rule("SQLCH220");
+    }
+    if (second === "SAMPLE" && u[2] === "BY") return rule("SQLCH217");
+    if (second === "TTL") return rule("SQLCH205");
+    if (second === "COMMENT") return rule("SQLCH203");
+    if (second === "QUERY") return rule("SQLCH241");
+    if (second === "REFRESH") return rule("SQLCH244");
+    if (second === "SETTING" || second === "SETTINGS") {
+      const key = (a[2] ?? "").toLowerCase();
+      return settingRule(key);
+    }
+    return { none: `ALTER TABLE ${u.slice(0, 2).join(" ")}` };
+  }
+  if (first === "REMOVE") {
+    if (second === "TTL") return rule("SQLCH205");
+    if (second === "SAMPLE" && u[2] === "BY") return rule("SQLCH217");
+    return { none: `ALTER TABLE REMOVE ${second ?? ""}`.trim() };
+  }
+  if (first === "UPDATE" || first === "DELETE") return { data: "a mutation: it rewrites the parts it touches in the background (system.mutations) and has no rollback" };
+  return { none: `ALTER TABLE ${u.slice(0, 2).join(" ")}` };
+}
+
+function settingRule(key: string): Reading {
+  if (STREAM_SETTING.test(key)) return rule("SQLCH225", { note: `${key} belongs to a stream engine, which has no MODIFY SETTING` });
+  const spec = (MERGE_TREE_SETTINGS as Record<string, { readonly: boolean } | undefined>)[key];
+  return spec?.readonly ? rule("SQLCH218", { note: `${key} is fixed at creation` }) : rule("SQLCH206");
+}
+
+/** The `ALTER TABLE` actions of one statement, in order, a `MODIFY SETTING` list kept together. */
+function alterTable(words: readonly string[], u: readonly string[], created: Map<string, Created>): Reading[] {
+  let i = 2;
+  if (u[i] === "IF" && u[i + 1] === "EXISTS") i += 2;
+  const { name, next } = nameAt(words, i);
+  i = next;
+  if (u[i] === "ON" && u[i + 1] === "CLUSTER") i += 3;
+  const actions = splitTopLevel(words.slice(i));
+  const added = new Set<string>();
+  for (const a of actions) {
+    const au = a.map((w) => w.toUpperCase());
+    if (au[0] === "ADD" && au[1] === "COLUMN") {
+      const k = au[2] === "IF" ? 5 : 2;
+      if (a[k]) added.add(bare(a[k]!));
+    }
+  }
+  const ctx: TableContext = { added, ...(created.has(bare(name)) ? { created: created.get(bare(name))! } : {}) };
+  const out: Reading[] = [];
+  let inSettings = false;
+  for (const a of actions) {
+    const au = a.map((w) => w.toUpperCase());
+    if (inSettings && au[0] !== undefined && !KNOWN_VERBS.has(au[0])) {
+      out.push(settingRule((a[0] ?? "").toLowerCase()));
+      continue;
+    }
+    inSettings = (au[0] === "MODIFY" || au[0] === "RESET") && (au[1] === "SETTING" || au[1] === "SETTINGS");
+    out.push(tableAction(a, ctx));
+  }
+  return out;
+}
+
+const OBJECTS: Record<string, ClassifierRuleId> = { USER: "SQLCH271", ROLE: "SQLCH270", QUOTA: "SQLCH275" };
+
+function alterOther(u: readonly string[]): Reading {
+  const obj = u[1];
+  if (obj && OBJECTS[obj]) return rule(OBJECTS[obj]!);
+  if (obj === "ROW" && u[2] === "POLICY") return rule("SQLCH272");
+  if (obj === "POLICY") return rule("SQLCH272");
+  if (obj === "SETTINGS" && u[2] === "PROFILE") return rule("SQLCH276");
+  if (obj === "DATABASE" && u.includes("COMMENT")) return rule("SQLCH203");
+  return { none: `ALTER ${obj ?? ""}`.trim() };
+}
+
+const DATA_WORDS = new Set(["INSERT", "SELECT", "WITH", "SYSTEM", "OPTIMIZE", "TRUNCATE", "SET", "USE", "EXPLAIN", "EXISTS", "DESCRIBE", "DESC", "SHOW", "CHECK", "KILL", "WATCH"]);
+
+
+/** A `CREATE` or `ATTACH`: the parser reads tables, views, dictionaries, databases and functions; the rest go by key words. */
+function create(tokens: Token[], u: readonly string[], created: Map<string, Created>): Reading {
+  let i = 1;
+  const orReplace = u[i] === "OR" && u[i + 1] === "REPLACE";
+  if (orReplace) i += 2;
+  let node: CreateNode | undefined;
+  try {
+    node = parseCreate(tokens);
+  } catch {
+    node = undefined;
+  }
+  if (node?.statement === "table") {
+    const name = text(tokens, node.name).replace(/\s*\.\s*/g, ".");
+    const keys = new Set<string>([...columnsOf(tokens, node.orderBy), ...columnsOf(tokens, node.primaryKey), ...columnsOf(tokens, node.partitionBy), ...columnsOf(tokens, node.sampleBy)]);
+    for (const c of node.columns) if (c.primaryKey) keys.add(bare(c.name));
+    created.set(bare(name), { keys, ...(node.orderBy ? { orderBy: keyText(text(tokens, node.orderBy).split(" ")) } : {}) });
+  }
+  if (orReplace && u[i] === "FUNCTION") return rule("SQLCH260");
+  if (orReplace && (u[i] === "TABLE" || node?.statement === "table")) return rule("SQLCH200", { note: "OR REPLACE drops an existing table of that name, and its rows, before it creates this one" });
+  return rule("SQLCH200");
+}
+
+function read(sql: string, created: Map<string, Created>): Reading[] {
+  let tokens: Token[];
+  try {
+    tokens = tokenizeText(sql, 0);
+  } catch {
+    return [{ none: "text the tokenizer could not read" }];
+  }
+  const words = tokens.filter((t) => !isTrivia(t) && t.text !== ";").map((t) => t.text);
+  const u = words.map((w) => w.toUpperCase());
+  const head = u[0];
+  if (head === undefined) return [{ data: "an empty statement" }];
+  if (head === "CREATE") return [create(tokens, u, created)];
+  if (head === "ALTER" && u[1] === "TABLE") return alterTable(words, u, created);
+  if (head === "ALTER") return [alterOther(u)];
+  if (head === "DROP") return [rule("SQLCH250", { destructive: true })];
+  if (head === "RENAME") {
+    if (u[1] === "DATABASE") return [rule("SQLCH231")];
+    if (u[1] === "TABLE" || u[1] === "DICTIONARY") return [rule("SQLCH230")];
+    return [{ none: `RENAME ${u[1] ?? ""}`.trim() }];
+  }
+  if (head === "GRANT") return [rule("SQLCH273")];
+  if (head === "REVOKE") return [rule("SQLCH274")];
+  if (head === "EXCHANGE") return [{ none: "EXCHANGE TABLES swaps two tables; no rule covers it" }];
+  if (head === "TRUNCATE") return [{ data: "TRUNCATE removes every row of the table" }];
+  if (DATA_WORDS.has(head)) return [{ data: `${head} reads or writes rows or the session, not the schema` }];
+  return [{ none: `${head} is not a statement the classifier reads` }];
+}
+
+/**
+ * Classify a migration's up statements, in order. The statements are the
+ * file's own text, one per entry.
+ */
+export function classifyClickHouseStatements(statements: readonly string[]): StatementVerdict[] {
+  const created = new Map<string, Created>();
+  return statements.map((sql, k) => {
+    const readings = read(sql, created);
+    const out: StatementVerdict = { index: k + 1, sql: sqlHead(sql), outcome: "classified", changes: [] };
+    const none: string[] = [];
+    const notes: string[] = [];
+    for (const r of readings) {
+      if ("rules" in r) out.changes.push(...r.rules.map((x) => classified(x.id, x.extra)));
+      else if ("undecided" in r) {
+        out.candidates = [...(out.candidates ?? []), ...r.undecided.map((id) => classified(id))];
+        notes.push(r.note);
+      } else if ("none" in r) none.push(r.none);
+      else notes.push(r.data);
+    }
+    if (out.candidates) out.outcome = "undecided";
+    else if (out.changes.length === 0 && none.length === 0) out.outcome = "data";
+    else if (none.length > 0) {
+      out.outcome = out.changes.length > 0 ? "classified" : "unclassified";
+      notes.push(`no rule covers ${none.join("; ")}`);
+    }
+    if (notes.length > 0) out.note = notes.join("; ");
+    return out;
+  });
+}

package/src/codegen/classifier-page.ts

@@ -93,6 +93,29 @@ A rebuild is never DDL: it is a step naming \`ClickHouseRebuildOp\`, or for a vi
 
 \`--topology\` renders the statements for where they will run: \`single\` (the default), \`cluster:<name>\`, \`replicated\` or \`cloud\`. On a cluster every statement carries \`ON CLUSTER\` and a MergeTree-family table becomes \`Replicated*MergeTree\` with its Keeper path; in a \`Replicated\` database the tables are \`Replicated*MergeTree\` with no path and no \`ON CLUSTER\`. \`diffStatements(before, after, { topology })\` takes the same choice; see [Applying to a Server](../applying/#topology).
 
+## Migration files on a pull request
+
+A workspace member that is a goose, golang-migrate, clickhouse-migrations or Flyway directory (see [Workspace kinds](/chant/reference/workspace-kinds/)) has no declaration to diff. On a pull request, \`chant components pr-plan\` lists each migration file the pull request adds to such a member, and the plan note gives every statement in it the class the rules below give the same change. This is the "check" level of adopting chant beside an existing migration tool: it reads files, applies nothing and contacts no server, so it needs no \`--env\`.
+
+The up statements are the ones \`chant sql replay\` would run: goose's \`-- +goose Up\` section, the whole \`.up.sql\` file for golang-migrate, the whole file for clickhouse-migrations, and a Flyway file after placeholder substitution with the values in \`flyway.toml\` or \`flyway.conf\`. A file the base commit already has is not new, whatever the pull request does to it. Undo files are left out.
+
+\`\`\`text
+\`db/00002_orders.sql\` (member \`ledger\`, postgres)
+- 1. \`CREATE INDEX orders_status_idx ON orders (status)\`: needs CONCURRENTLY (SQLPG241, SHARE)
+- 2. \`ALTER TABLE orders ALTER COLUMN a TYPE bigint\`: undecided between SQLPG206 metadata only and SQLPG207 ACCESS EXCLUSIVE rewrite or scan
+\`\`\`
+
+Each statement is read with the dialect's parser (\`CREATE\`, \`GRANT\`, \`COMMENT ON\`) or word by word (\`ALTER\`, \`DROP\`, \`RENAME\`), and the lock on a Postgres statement is the one its rule's text names. The code is \`lexicons/sql/src/postgres/plan/statements.ts\` and \`lexicons/sql/src/clickhouse/plan/statements.ts\`.
+
+A statement alone cannot always decide a rule, and then the note names the candidates and does not guess:
+
+- \`ALTER COLUMN ... TYPE\` is SQLPG206 or SQLPG207 by the column's old type, and SQLPG208 as well with a \`USING\` expression.
+- \`MODIFY COLUMN <type>\` is SQLCH210 or SQLCH211, \`DROP COLUMN\` SQLCH202 or SQLCH213, and \`RENAME COLUMN\` SQLCH212 or SQLCH213, by whether the column is in a key. When the same file creates the table, its keys are known and the statement is classified.
+- \`MODIFY ORDER BY\` is SQLCH220, a rebuild. It is SQLCH216 or SQLCH220 when the same \`ALTER\` adds the columns the new key ends with, unless the file created the table and shows the old key is kept.
+- \`CREATE OR REPLACE VIEW\` is SQLPG250 or SQLPG251 by whether the view keeps its columns.
+
+A statement that reads or writes rows (\`INSERT\`, \`UPDATE\`, \`TRUNCATE\`) is listed as data. A schema change no rule covers, such as \`ALTER TABLE ... OWNER TO\` or a ClickHouse \`DROP PARTITION\`, is listed as having no class.
+
 ## Identity and renames
 
 An object is identified by its export name between two builds, so a changed name in the SQL under the same export is a rename. Against a server, which has no export names, an object is identified by \`database.name\`; a declaration renamed since the server last saw it says so with \`-- previously: <old name>\` before its CREATE.

package/src/codegen/postgres-classifier-page.ts

@@ -65,14 +65,14 @@ A column rename or a type change across kinds is never DDL: it is a step naming
 
 ## Which major
 
-Two rules change class across the supported majors: a STORED generated column's expression (SQLPG212) is a rewrite from 17 and expand and contract before it, and a table's access method (SQLPG226) is a rewrite from 15 and expand and contract before it.
+Three rules change class across the supported majors: a STORED generated column's expression (SQLPG212) is a rewrite from 17 and expand and contract before it, a table's access method (SQLPG226) is a rewrite from 15 and expand and contract before it, and an operator's commutator, negator, HASHES and MERGES (SQLPG304) are altered in place from 17 and dropped and created before it.
 
 - \`chant sql diff\` classifies for the major the builds recorded (\`postgresMajor\` in the build output): the newer build's, else the older build's, else \`sql.postgresMajor\`, else 18. A diff between two revisions does not depend on today's config.
 - \`chant sql plan\` classifies for the major the server runs (\`server_version_num\`), since the server is what takes the locks. When the build targets another major, the plan says so in a hint.
 
 ## Identity and renames
 
-An object is identified by its export name between two builds, and by its schema-qualified name against a server, which has no export names. Tables, views, materialized views, sequences and indexes share one namespace per schema, types and domains another, and schemas and extensions are matched by name. Functions, procedures and aggregates share a namespace of their own and are told apart by their input parameter types, so an overload is another object; composite and range types share the types' namespace, collations have one of their own, extended statistics another, an operator by its symbol and operand types, a cast by its two types and an event trigger, a foreign data wrapper, a foreign server and a publication by their names in the database; a trigger is matched by its name on its table.
+An object is identified by its export name between two builds, and by its schema-qualified name against a server, which has no export names. Tables, foreign tables, views, materialized views, sequences and indexes share one namespace per schema, types and domains another, and schemas and extensions are matched by name. Functions, procedures and aggregates share a namespace of their own and are told apart by their input parameter types, so an overload is another object; composite and range types share the types' namespace, collations have one of their own, extended statistics another, an operator by its symbol and operand types, an operator class and an operator family each by its name and index method, text search configurations one namespace and dictionaries another, a cast by its two types and an event trigger, a foreign data wrapper, a foreign server and a publication by their names in the database; a trigger is matched by its name on its table.
 
 A rename is declared where it happens. Before a CREATE, \`-- previously: <old name>\` says the object had another name; on a column's line, it says the column did:
 

package/src/codegen/registry.ts

@@ -40,6 +40,11 @@ const KINDS: Record<string, string> = {
   PostgresCast: POSTGRES_ENTITY_TYPES.cast,
   PostgresEventTrigger: POSTGRES_ENTITY_TYPES.eventTrigger,
   PostgresOperator: POSTGRES_ENTITY_TYPES.operator,
+  PostgresOperatorClass: POSTGRES_ENTITY_TYPES.operatorClass,
+  PostgresOperatorFamily: POSTGRES_ENTITY_TYPES.operatorFamily,
+  PostgresForeignTable: POSTGRES_ENTITY_TYPES.foreignTable,
+  PostgresTextSearchConfiguration: POSTGRES_ENTITY_TYPES.textSearchConfiguration,
+  PostgresTextSearchDictionary: POSTGRES_ENTITY_TYPES.textSearchDictionary,
   PostgresStatistics: POSTGRES_ENTITY_TYPES.statistics,
   PostgresForeignDataWrapper: POSTGRES_ENTITY_TYPES.foreignDataWrapper,
   PostgresForeignServer: POSTGRES_ENTITY_TYPES.foreignServer,

package/src/core/statement-class.ts

@@ -0,0 +1,88 @@
+/**
+ * What the statement classifiers (`../clickhouse/plan/statements.ts`,
+ * `../postgres/plan/statements.ts`) answer, in one shape for both dialects
+ * (#4407). They read a migration file's statements offline and give each the
+ * class the schema-diff classifier gives the same change, citing its rule.
+ * Nothing runs against a server.
+ */
+
+import type { ClassifierRule } from "./classifier";
+
+/** One change a statement makes, under one rule. */
+export interface StatementClassified {
+  rule: string;
+  /** The rule's class, e.g. `rebuild`, `concurrently`. */
+  class: string;
+  /** The class as a report names it, e.g. `REBUILD`. */
+  label: string;
+  title: string;
+  /** Postgres: the lock the rule's restriction names, e.g. `SHARE`. */
+  lock?: string;
+  /** Data is removed and not recoverable. */
+  destructive?: boolean;
+  note?: string;
+}
+
+/**
+ * - `classified`: every change has a rule.
+ * - `undecided`: the statement alone cannot say which of `candidates` applies
+ *   (a column type change is a rewrite, or a rebuild when the column is in a
+ *   key); the answer `classifyDisruption()` calls `unknown`.
+ * - `unclassified`: a schema change no rule covers; `note` says what.
+ * - `data`: the statement reads or writes rows or the session, not the schema.
+ */
+export type StatementOutcome = "classified" | "undecided" | "unclassified" | "data";
+
+export interface StatementVerdict {
+  /** 1-based position among the file's up statements. */
+  index: number;
+  /** The statement's first line, cut to 120 characters. */
+  sql: string;
+  outcome: StatementOutcome;
+  /** `classified`: one entry per change in the statement (an `ALTER TABLE` can hold several). */
+  changes: StatementClassified[];
+  /** `undecided`: the rules that could apply, in the order a reader should weigh them. */
+  candidates?: StatementClassified[];
+  note?: string;
+}
+
+/** A statement's own text cut for a note: the first line, at most 120 characters. */
+export function sqlHead(sql: string): string {
+  const line = sql.split(/\r?\n/).find((l) => l.trim() !== "" && !l.trim().startsWith("--")) ?? sql;
+  const t = line.trim().replace(/\s+/g, " ");
+  return t.length > 120 ? `${t.slice(0, 117)}...` : t;
+}
+
+/** The pieces of a classified change from a dialect's rule table. */
+export function classifiedOf<C extends string>(
+  rules: Readonly<Record<string, ClassifierRule<C>>>,
+  labels: Readonly<Record<C, string>>,
+  id: string,
+  extra: Partial<StatementClassified> = {},
+): StatementClassified {
+  const r = rules[id];
+  if (!r) throw new Error(`no classifier rule ${id}`);
+  return { rule: id, class: r.class, label: labels[r.class], title: r.title, ...extra };
+}
+
+/** Lock names, longest first so `SHARE UPDATE EXCLUSIVE` is not read as `SHARE`. */
+const LOCK = /\b(ACCESS EXCLUSIVE|SHARE UPDATE EXCLUSIVE|SHARE ROW EXCLUSIVE|ROW EXCLUSIVE|ROW SHARE|EXCLUSIVE|SHARE)\b/;
+
+/** The first lock a rule's restriction names, or undefined when it names none. */
+export const lockOf = (restriction: string): string | undefined => LOCK.exec(restriction)?.[1];
+
+/**
+ * Split a statement's tokens' text into the pieces between top-level commas
+ * (an `ALTER TABLE`'s actions). Parentheses and brackets nest.
+ */
+export function splitTopLevel(words: readonly string[]): string[][] {
+  const out: string[][] = [[]];
+  let depth = 0;
+  for (const w of words) {
+    if (w === "(" || w === "[") depth++;
+    else if (w === ")" || w === "]") depth--;
+    if (w === "," && depth === 0) out.push([]);
+    else out[out.length - 1]!.push(w);
+  }
+  return out.filter((a) => a.length > 0);
+}

package/src/coverage/postgres.json

@@ -11,16 +11,16 @@
     { "command": "CREATE EVENT TRIGGER", "kind": "event trigger", "status": "declared", "entity": "Postgres::EventTrigger", "note": "the database's, not a schema's; creating one takes a superuser" },
     { "command": "CREATE EXTENSION", "kind": "extension", "status": "declared", "entity": "Postgres::Extension" },
     { "command": "CREATE FOREIGN DATA WRAPPER", "kind": "foreign data wrapper", "status": "declared", "entity": "Postgres::ForeignDataWrapper", "note": "the database's, not a schema's; creating one takes a superuser; one an extension installs comes with the extension; an option that holds a credential is refused" },
-    { "command": "CREATE FOREIGN TABLE", "kind": "foreign table", "status": "refused", "reason": "no tag declares a foreign table yet" },
+    { "command": "CREATE FOREIGN TABLE", "kind": "foreign table", "status": "declared", "entity": "Postgres::ForeignTable", "note": "its own tag, not table: another CREATE and ALTER, no key, index or storage clauses; its server cannot be altered; one that is a partition, inherits or has a generated column is not declared; an option that holds a credential is refused" },
     { "command": "CREATE FUNCTION", "kind": "function", "status": "declared", "entity": "Postgres::Function" },
     { "command": "CREATE FUNCTION", "form": "RETURN or BEGIN ATOMIC body", "match": "\\b(?:RETURN|BEGIN\\s+ATOMIC)\\b", "kind": "routine with a SQL-standard body", "status": "refused", "reason": "the server stores a SQL-standard body parsed and prints it back rewritten, so a declaration cannot be compared with it; declare the routine with a string body (AS $$ ... $$)" },
     { "command": "CREATE GROUP", "kind": "group", "status": "declared", "entity": "Postgres::Role", "note": "an alias of CREATE ROLE" },
     { "command": "CREATE INDEX", "kind": "index", "status": "declared", "entity": "Postgres::Index" },
     { "command": "CREATE LANGUAGE", "kind": "procedural language", "status": "refused", "reason": "no tag declares a procedural language; an extension installs one, and the extension is declared instead" },
     { "command": "CREATE MATERIALIZED VIEW", "kind": "materialized view", "status": "declared", "entity": "Postgres::MaterializedView" },
-    { "command": "CREATE OPERATOR", "kind": "operator", "status": "declared", "entity": "Postgres::Operator", "note": "identified by its name and operand types; a change is a drop and a create" },
-    { "command": "CREATE OPERATOR CLASS", "kind": "operator class", "status": "refused", "reason": "no tag declares an operator class yet" },
-    { "command": "CREATE OPERATOR FAMILY", "kind": "operator family", "status": "refused", "reason": "no tag declares an operator family yet" },
+    { "command": "CREATE OPERATOR", "kind": "operator", "status": "declared", "entity": "Postgres::Operator", "note": "identified by its name and operand types; RESTRICT and JOIN change in place, and from 17 so does setting a commutator or negator or turning on HASHES or MERGES; anything else is a drop and a create" },
+    { "command": "CREATE OPERATOR CLASS", "kind": "operator class", "status": "declared", "entity": "Postgres::OperatorClass", "note": "identified by its name and index method; a change is a drop and a create, since ALTER OPERATOR CLASS only renames or moves it; its own family, named as the class, comes with it" },
+    { "command": "CREATE OPERATOR FAMILY", "kind": "operator family", "status": "declared", "entity": "Postgres::OperatorFamily", "note": "identified by its name and index method; its members come with the classes that name it, and loose members added with ALTER OPERATOR FAMILY are not declared" },
     { "command": "CREATE POLICY", "kind": "policy", "status": "declared", "entity": "Postgres::Policy", "note": "read where the profile manages access (sql.profiles.<env>.access)" },
     { "command": "CREATE PROCEDURE", "kind": "procedure", "status": "declared", "entity": "Postgres::Procedure" },
     { "command": "CREATE PUBLICATION", "kind": "publication", "status": "declared", "entity": "Postgres::Publication", "note": "the database's, not a schema's; a table of it may carry a column list and row filter from 15" },
@@ -34,8 +34,8 @@
     { "command": "CREATE TABLE", "kind": "table", "status": "declared", "entity": "Postgres::Table" },
     { "command": "CREATE TABLE AS", "kind": "table from a query", "status": "declared", "entity": "Postgres::Table", "note": "the server keeps no query, so the table it made reads as a CREATE TABLE" },
     { "command": "CREATE TABLESPACE", "kind": "tablespace", "status": "out-of-scope", "reason": "a tablespace is the cluster's, a directory on the server; a table's TABLESPACE clause is kept" },
-    { "command": "CREATE TEXT SEARCH CONFIGURATION", "kind": "text search configuration", "status": "refused", "reason": "no tag declares a text search configuration yet" },
-    { "command": "CREATE TEXT SEARCH DICTIONARY", "kind": "text search dictionary", "status": "refused", "reason": "no tag declares a text search dictionary yet" },
+    { "command": "CREATE TEXT SEARCH CONFIGURATION", "kind": "text search configuration", "status": "declared", "entity": "Postgres::TextSearchConfiguration", "note": "PARSER and its ADD MAPPING statements; COPY is refused, since it copies mappings chant does not read; mappings change in place, the parser by drop and create" },
+    { "command": "CREATE TEXT SEARCH DICTIONARY", "kind": "text search dictionary", "status": "declared", "entity": "Postgres::TextSearchDictionary", "note": "a template and its options; options change in place, the template by drop and create" },
     { "command": "CREATE TEXT SEARCH PARSER", "kind": "text search parser", "status": "refused", "reason": "no tag declares a text search parser; one is C code an extension installs" },
     { "command": "CREATE TEXT SEARCH TEMPLATE", "kind": "text search template", "status": "refused", "reason": "no tag declares a text search template; one is C code an extension installs" },
     { "command": "CREATE TRANSFORM", "kind": "transform", "status": "refused", "reason": "no tag declares a transform; an extension installs one" },

package/src/drizzle.ts

@@ -80,6 +80,11 @@ const SKIP_REASONS: Record<string, string> = {
   "Postgres::Collation": "Drizzle has no collation",
   "Postgres::Cast": "Drizzle has no cast",
   "Postgres::Operator": "Drizzle has no operator",
+  "Postgres::OperatorClass": "Drizzle has no operator class",
+  "Postgres::OperatorFamily": "Drizzle has no operator family",
+  "Postgres::ForeignTable": "Drizzle has no foreign table",
+  "Postgres::TextSearchConfiguration": "Drizzle has no text search configuration",
+  "Postgres::TextSearchDictionary": "Drizzle has no text search dictionary",
   "Postgres::Statistics": "Drizzle has no extended statistics",
   "Postgres::ForeignDataWrapper": "Drizzle has no foreign data wrapper",
   "Postgres::ForeignServer": "Drizzle has no foreign server",

package/src/files/postgres.ts

@@ -150,6 +150,10 @@ export function readPostgres(ddl: string, options: ReadOptions, problems: string
         mark(node.table);
         followOns.push({ tokens, node, inserts });
         continue;
+      case "textSearchMapping":
+        mark(node.name);
+        followOns.push({ tokens, node, inserts });
+        continue;
       case "index":
       case "trigger":
       case "policy":
@@ -189,7 +193,7 @@ export function readPostgres(ddl: string, options: ReadOptions, problems: string
     const k = followOnOwner(f.node, f.tokens, nameOf, qualify);
     const owner = k ? owners.get(k) : undefined;
     if (!owner) {
-      const what = f.node.statement === "comment" ? `COMMENT ON ${f.node.objectType}` : "ALTER TABLE ... ROW LEVEL SECURITY";
+      const what = f.node.statement === "comment" ? `COMMENT ON ${f.node.objectType}` : f.node.statement === "textSearchMapping" ? "ALTER TEXT SEARCH CONFIGURATION ... ADD MAPPING" : "ALTER TABLE ... ROW LEVEL SECURITY";
       problems.push(`${what} for an object the file does not create: ${firstLine(tokensText(f.tokens))}`);
       continue;
     }
@@ -283,11 +287,19 @@ function ownerKey(node: StatementNode, tokens: Token[], nameOf: NameOf, qualify:
     case "foreignServer":
     case "publication":
       return `${node.statement} ${nameOf(tokens, node.name).at(-1) ?? ""}`;
+    // A class or family is named by its index method too, and a configuration and a dictionary are different namespaces.
+    case "operatorClass":
+    case "operatorFamily":
+      return `${node.statement} ${qkey(qualify(nameOf(tokens, node.name)))} USING ${node.method}`;
+    case "textSearchConfiguration":
+    case "textSearchDictionary":
+      return `${node.statement} ${qkey(qualify(nameOf(tokens, node.name)))}`;
     case "cast":
       return `cast ${castKey(tokens, node.source, node.target)}`;
     case "grant":
     case "comment":
     case "rowSecurity":
+    case "textSearchMapping":
       return undefined;
     default:
       return `rel ${qkey(qualify(nameOf(tokens, node.name)))}`;
@@ -297,6 +309,7 @@ function ownerKey(node: StatementNode, tokens: Token[], nameOf: NameOf, qualify:
 /** The key of the object a COMMENT ON or ALTER TABLE ... ROW LEVEL SECURITY finishes. */
 function followOnOwner(node: StatementNode, tokens: Token[], nameOf: NameOf, qualify: Qualify): string | undefined {
   if (node.statement === "rowSecurity") return `rel ${qkey(qualify(nameOf(tokens, node.table)))}`;
+  if (node.statement === "textSearchMapping") return `textSearchConfiguration ${qkey(qualify(nameOf(tokens, node.name)))}`;
   if (node.statement !== "comment") return undefined;
   const target = nameOf(tokens, node.target);
   switch (node.objectType) {
@@ -321,6 +334,14 @@ function followOnOwner(node: StatementNode, tokens: Token[], nameOf: NameOf, qua
       return `foreignServer ${target.at(-1) ?? ""}`;
     case "PUBLICATION":
       return `publication ${target.at(-1) ?? ""}`;
+    case "OPERATOR CLASS":
+      return `operatorClass ${qkey(qualify(target))} USING ${node.using ?? ""}`;
+    case "OPERATOR FAMILY":
+      return `operatorFamily ${qkey(qualify(target))} USING ${node.using ?? ""}`;
+    case "TEXT SEARCH CONFIGURATION":
+      return `textSearchConfiguration ${qkey(qualify(target))}`;
+    case "TEXT SEARCH DICTIONARY":
+      return `textSearchDictionary ${qkey(qualify(target))}`;
     case "CAST":
       return node.cast ? `cast ${castKey(tokens, node.cast.source, node.cast.target)}` : undefined;
     case "CONSTRAINT":

package/src/generated/lexicon-sql.json

@@ -134,6 +134,31 @@
     "kind": "resource",
     "lexicon": "sql"
   },
+  "PostgresOperatorClass": {
+    "resourceType": "Postgres::OperatorClass",
+    "kind": "resource",
+    "lexicon": "sql"
+  },
+  "PostgresOperatorFamily": {
+    "resourceType": "Postgres::OperatorFamily",
+    "kind": "resource",
+    "lexicon": "sql"
+  },
+  "PostgresForeignTable": {
+    "resourceType": "Postgres::ForeignTable",
+    "kind": "resource",
+    "lexicon": "sql"
+  },
+  "PostgresTextSearchConfiguration": {
+    "resourceType": "Postgres::TextSearchConfiguration",
+    "kind": "resource",
+    "lexicon": "sql"
+  },
+  "PostgresTextSearchDictionary": {
+    "resourceType": "Postgres::TextSearchDictionary",
+    "kind": "resource",
+    "lexicon": "sql"
+  },
   "PostgresStatistics": {
     "resourceType": "Postgres::Statistics",
     "kind": "resource",

package/src/index.ts

@@ -18,7 +18,7 @@ export { database, table, view, dictionary, user, role, policy, quota, settingsP
 // A tag folds as the function its file imports, so `table` imported from
 // `@intentius/chant-lexicon-sql/postgres` is Postgres's. Declarations import
 // them from the dialect subpath.
-export { schema, index, sequence, type, domain, extension, func, procedure, aggregate, trigger, eventTrigger, operator, statistics, foreignDataWrapper, foreignServer, publication, collation, cast } from "./postgres/entities";
+export { schema, index, sequence, type, domain, extension, func, procedure, aggregate, trigger, eventTrigger, operator, operatorClass, operatorFamily, foreignTable, textSearchConfiguration, textSearchDictionary, statistics, foreignDataWrapper, foreignServer, publication, collation, cast } from "./postgres/entities";
 
 // A schema change's statements between two builds, rendered offline for a
 // migration file (#3644); also at `@intentius/chant-lexicon-sql/migration-statements`.