@intentius/chant-lexicon-sql 0.124.46 → 0.124.49

package/src/postgres/plan/statements.ts

@@ -0,0 +1,299 @@
+/**
+ * The Postgres statement classifier (#4407): a migration file's statements,
+ * each given the class and rule the schema-diff classifier (`./rules.ts`)
+ * gives the same change, with the lock the rule's restriction names. It reads
+ * text and contacts no server.
+ *
+ * `CREATE`, `GRANT`, `REVOKE`, `COMMENT ON` and `ALTER TABLE ... ROW LEVEL
+ * SECURITY` go through the declaration parser (`../parser.ts`). The parser
+ * has no `ALTER` grammar, so the other `ALTER` and `DROP` statements are read
+ * word by word from the same tokens.
+ *
+ * A statement alone cannot decide some changes. `ALTER COLUMN ... TYPE` is a
+ * catalog change or a table rewrite by the column's old type, which the file
+ * does not state: it is `undecided`, naming SQLPG206 and SQLPG207 (and
+ * SQLPG208 with a `USING` expression). `CREATE OR REPLACE VIEW` is SQLPG250
+ * or SQLPG251 by whether the columns stay. A statement no rule covers is
+ * `unclassified` with a note.
+ */
+
+import { isTrivia, tokenizeText, type Token } from "../tokens";
+import { parseStatements, type StatementNode } from "../parser";
+import { classifiedOf, lockOf, splitTopLevel, sqlHead, type StatementClassified, type StatementVerdict } from "../../core/statement-class";
+import { PG_CHANGE_CLASSES, PG_CLASSIFIER_RULES, type PgClassifierRuleId } from "./rules";
+
+const classified = (id: PgClassifierRuleId, extra: Partial<StatementClassified> = {}): StatementClassified => {
+  const lock = lockOf(PG_CLASSIFIER_RULES[id].restriction);
+  return classifiedOf(PG_CLASSIFIER_RULES, PG_CHANGE_CLASSES.label, id, { ...(lock ? { lock } : {}), ...extra });
+};
+
+/** What one clause of a statement comes to. */
+type Reading = { rules: Array<{ id: PgClassifierRuleId; extra?: Partial<StatementClassified> }> } | { undecided: PgClassifierRuleId[]; note: string } | { none: string } | { data: string };
+
+const rule = (id: PgClassifierRuleId, extra?: Partial<StatementClassified>): Reading => ({ rules: [{ id, ...(extra ? { extra } : {}) }] });
+
+const VOLATILE = /\b(?:clock_timestamp|random|gen_random_uuid|uuid_generate_v\d\w*|nextval|timeofday|txid_current|setseed)\s*\(/i;
+
+/** The words from `i` that are one possibly qualified name, and the index after it. */
+function nameAt(words: readonly string[], i: number): { name: string; next: number } {
+  let name = words[i] ?? "";
+  let j = i + 1;
+  while (words[j] === "." && words[j + 1] !== undefined) {
+    name += `.${words[j + 1]}`;
+    j += 2;
+  }
+  return { name, next: j };
+}
+
+const bare = (name: string): string => (name.split(".").pop() ?? name).replace(/^"|"$/g, "").toLowerCase();
+
+const has = (u: readonly string[], ...seq: string[]): boolean => u.some((_, i) => seq.every((w, k) => u[i + k] === w));
+
+/** One `ALTER TABLE` action (the words between top-level commas). */
+function tableAction(a: readonly string[]): Reading {
+  const u = a.map((w) => w.toUpperCase());
+  const first = u[0];
+  if (first === "ADD") {
+    let i = 1;
+    if (u[i] === "COLUMN") i++;
+    if (u[i] === "CONSTRAINT" || ["PRIMARY", "UNIQUE", "CHECK", "FOREIGN", "EXCLUDE"].includes(u[i] ?? "")) {
+      let k = i;
+      if (u[k] === "CONSTRAINT") k += 2;
+      const kind = u[k];
+      const notValid = has(u, "NOT", "VALID");
+      if (kind === "NOT" && u[k + 1] === "NULL") return rule("SQLPG210");
+      if (notValid) return rule("SQLPG217", { note: `${kind ?? "constraint"} added NOT VALID; VALIDATE CONSTRAINT is SQLPG220` });
+      if (kind === "PRIMARY" || kind === "UNIQUE") return rule("SQLPG221", has(u, "USING", "INDEX") ? { note: "from an existing unique index, so no build under this lock" } : {});
+      if (kind === "CHECK") return rule("SQLPG218");
+      if (kind === "FOREIGN") return rule("SQLPG219");
+      if (kind === "EXCLUDE") return rule("SQLPG222");
+      return { none: "an ADD CONSTRAINT of a kind no rule names" };
+    }
+    if (u[i] === "IF" && u[i + 1] === "NOT" && u[i + 2] === "EXISTS") i += 3;
+    const hasDefault = u.includes("DEFAULT");
+    const defaultText = hasDefault ? a.slice(u.indexOf("DEFAULT") + 1).join(" ") : "";
+    const generated = has(u, "GENERATED");
+    const stored = u.includes("STORED") && generated;
+    const identity = u.includes("IDENTITY");
+    const notNull = has(u, "NOT", "NULL");
+    if (stored || identity) return rule("SQLPG202", { note: stored ? "a STORED generated column" : "an identity column" });
+    if (VOLATILE.test(defaultText)) return rule("SQLPG202", { note: "a volatile default" });
+    if (notNull && !hasDefault && !generated) return rule("SQLPG203");
+    return rule("SQLPG201");
+  }
+  if (first === "DROP") {
+    if (u[1] === "CONSTRAINT") return rule("SQLPG223");
+    return rule("SQLPG204", { destructive: true });
+  }
+  if (first === "RENAME") {
+    if (u[1] === "COLUMN" || (u[1] !== "CONSTRAINT" && u[1] !== "TO")) return rule("SQLPG205");
+    return rule("SQLPG228");
+  }
+  if (first === "ALTER") {
+    let i = 1;
+    if (u[i] === "COLUMN") i++;
+    if (u[i] === "CONSTRAINT") return { none: "ALTER CONSTRAINT (deferrability)" };
+    const rest = u.slice(i + 1);
+    const col = a[i] ?? "column";
+    if (rest[0] === "TYPE" || (rest[0] === "SET" && rest[1] === "DATA" && rest[2] === "TYPE")) {
+      const using = rest.includes("USING");
+      return {
+        undecided: using ? ["SQLPG206", "SQLPG207", "SQLPG208"] : ["SQLPG206", "SQLPG207"],
+        note: `the new type of ${col} is a catalog change or a table rewrite by its old type${using ? ", or a change across kinds with this USING expression," : ""}, which the file does not state`,
+      };
+    }
+    if (rest[0] === "SET" && rest[1] === "DEFAULT") return rule("SQLPG209");
+    if (rest[0] === "DROP" && rest[1] === "DEFAULT") return rule("SQLPG209");
+    if (rest[0] === "SET" && rest[1] === "NOT" && rest[2] === "NULL") return rule("SQLPG210");
+    if (rest[0] === "DROP" && rest[1] === "NOT" && rest[2] === "NULL") return rule("SQLPG211");
+    if (rest[0] === "SET" && rest[1] === "EXPRESSION") return rule("SQLPG212");
+    if ((rest[0] === "ADD" && rest[1] === "GENERATED") || (rest[0] === "SET" && (rest[1] === "GENERATED" || rest[1] === "INCREMENT" || rest[1] === "START" || rest[1] === "RESTART")) || rest[0] === "RESTART" || (rest[0] === "DROP" && rest[1] === "IDENTITY")) {
+      return rule("SQLPG213");
+    }
+    if (rest[0] === "SET" && (rest[1] === "STORAGE" || rest[1] === "COMPRESSION")) return rule("SQLPG215");
+    return { none: `ALTER COLUMN ${rest.slice(0, 2).join(" ")}` };
+  }
+  if (first === "VALIDATE" && u[1] === "CONSTRAINT") return rule("SQLPG220");
+  if (first === "SET" || first === "RESET") {
+    if (u[1] === "LOGGED" || u[1] === "UNLOGGED") return rule("SQLPG225");
+    if (u[1] === "ACCESS" || u[1] === "TABLESPACE") return rule("SQLPG226");
+    if (u[1] === "SCHEMA") return rule("SQLPG228");
+    if (u[1] === "(" || first === "RESET") return rule("SQLPG224");
+    return { none: `ALTER TABLE ${u.slice(0, 2).join(" ")}` };
+  }
+  if (first === "ATTACH" || first === "DETACH" || first === "INHERIT" || (first === "NO" && u[1] === "INHERIT") || first === "OF" || (first === "NOT" && u[1] === "OF")) return rule("SQLPG227");
+  if ((first === "ENABLE" || first === "DISABLE" || first === "FORCE" || (first === "NO" && u[1] === "FORCE")) && has(u, "ROW", "LEVEL", "SECURITY")) return rule("SQLPG293");
+  if ((first === "ENABLE" || first === "DISABLE") && (u[1] === "TRIGGER" || u[1] === "REPLICA" || u[1] === "ALWAYS")) return rule("SQLPG284");
+  return { none: `ALTER TABLE ${u.slice(0, 2).join(" ")}` };
+}
+
+/** `ALTER <object> ...` other than `TABLE`, by its second word. */
+function alterOther(u: readonly string[], a: readonly string[]): Reading {
+  const obj = u[1];
+  const rename = u.includes("RENAME");
+  if (obj === "INDEX") return rename ? rule("SQLPG229") : rule("SQLPG243");
+  if (obj === "TYPE") {
+    if (has(u, "ADD", "VALUE")) return rule("SQLPG260");
+    if (has(u, "RENAME", "TO")) return rule("SQLPG228");
+    return { none: "ALTER TYPE other than ADD VALUE and RENAME TO" };
+  }
+  if (obj === "SEQUENCE") return rename || has(u, "SET", "SCHEMA") ? rule("SQLPG228") : has(u, "OWNER", "TO") ? { none: "ALTER SEQUENCE OWNER TO" } : rule("SQLPG265");
+  if (obj === "VIEW") return rename || has(u, "SET", "SCHEMA") ? rule("SQLPG228") : has(u, "OWNER", "TO") ? { none: "ALTER VIEW OWNER TO" } : rule("SQLPG253");
+  if (obj === "MATERIALIZED") return rename || has(u, "SET", "SCHEMA") ? rule("SQLPG228") : { none: "ALTER MATERIALIZED VIEW other than RENAME" };
+  if (obj === "SCHEMA") return rename ? rule("SQLPG228") : has(u, "OWNER", "TO") ? rule("SQLPG267") : { none: "ALTER SCHEMA" };
+  if (obj === "EXTENSION") return has(u, "UPDATE") || has(u, "SET", "SCHEMA") ? rule("SQLPG266") : { none: "ALTER EXTENSION ADD or DROP member" };
+  if (obj === "FUNCTION" || obj === "PROCEDURE" || obj === "ROUTINE") return rename || has(u, "SET", "SCHEMA") ? rule("SQLPG228") : { none: `ALTER ${obj} other than RENAME` };
+  if (obj === "TRIGGER") return rule("SQLPG284");
+  if (obj === "POLICY") return rename ? rule("SQLPG228") : rule("SQLPG291");
+  if (obj === "ROLE" || obj === "USER" || obj === "GROUP") return rule("SQLPG294");
+  if (obj === "DOMAIN") {
+    if (rename || has(u, "SET", "SCHEMA")) return rule("SQLPG228");
+    if (has(u, "ADD", "CONSTRAINT") || (u[3] === "ADD" && ["CHECK", "NOT"].includes(u[4] ?? ""))) return rule("SQLPG262");
+    if (has(u, "OWNER", "TO")) return { none: "ALTER DOMAIN OWNER TO" };
+    return rule("SQLPG263");
+  }
+  if (obj === "SERVER" || obj === "PUBLICATION" || (obj === "FOREIGN" && u[2] === "DATA")) return rename ? rule("SQLPG228") : rule("SQLPG303");
+  void a;
+  return { none: `ALTER ${obj ?? ""}`.trim() };
+}
+
+function dropStatement(u: readonly string[]): Reading {
+  const obj = u[1] === "MATERIALIZED" || u[1] === "FOREIGN" || u[1] === "EVENT" ? `${u[1]} ${u[2]}` : u[1];
+  if (obj === "INDEX") return rule("SQLPG242", u[2] === "CONCURRENTLY" ? { note: "DROP INDEX CONCURRENTLY waits for running queries instead of blocking them" } : {});
+  if (obj === "TRIGGER") return rule("SQLPG285", { destructive: false });
+  if (obj === "POLICY") return rule("SQLPG292", { destructive: false });
+  return rule("SQLPG270", { destructive: true });
+}
+
+/** How a `CREATE` that the parser could not read starts: its kind by key words. */
+function createByWords(u: readonly string[]): Reading {
+  let i = 1;
+  const orReplace = u[i] === "OR" && u[i + 1] === "REPLACE";
+  if (orReplace) i += 2;
+  while (["GLOBAL", "LOCAL", "TEMPORARY", "TEMP", "UNLOGGED", "RECURSIVE", "UNIQUE", "CONSTRAINT"].includes(u[i] ?? "") && u[i + 1] !== "INDEX" && u[i] !== "UNIQUE") i++;
+  if (u[i] === "UNIQUE") i++;
+  if (u[i] === "INDEX") {
+    const concurrently = u[i + 1] === "CONCURRENTLY";
+    return concurrently ? rule("SQLPG240") : rule("SQLPG241");
+  }
+  if (orReplace && (u[i] === "FUNCTION" || u[i] === "PROCEDURE")) return rule("SQLPG280");
+  if (orReplace && u[i] === "VIEW") return { undecided: ["SQLPG250", "SQLPG251"], note: "whether the view keeps its columns depends on the view it replaces" };
+  if (orReplace && u[i] === "AGGREGATE") return rule("SQLPG302");
+  if (u[i] === "TRIGGER") return rule("SQLPG283");
+  if (u[i] === "POLICY") return rule("SQLPG290");
+  return rule("SQLPG200");
+}
+
+/** A statement the parser read: its rule, given the tables this file created earlier. */
+function fromNode(n: StatementNode, created: Set<string>): Reading {
+  switch (n.statement) {
+    case "table":
+      created.add(bare(n.name.pieces.join(".")));
+      return rule("SQLPG200");
+    case "index":
+      if (n.concurrently) return rule("SQLPG240");
+      if (created.has(bare(n.table.pieces.join(".")))) return rule("SQLPG200", { note: "an index on a table this file creates, which has no rows to block" });
+      return rule("SQLPG241");
+    case "view":
+      if (n.orReplace && !n.materialized) return { undecided: ["SQLPG250", "SQLPG251"], note: "whether the view keeps its columns depends on the view it replaces" };
+      return rule("SQLPG200");
+    case "function":
+    case "procedure":
+      return n.orReplace ? rule("SQLPG280") : rule("SQLPG200");
+    case "aggregate":
+      return n.orReplace ? rule("SQLPG302") : rule("SQLPG200");
+    case "trigger":
+      return created.has(bare(n.table.pieces.join("."))) ? rule("SQLPG200", { note: "a trigger on a table this file creates" }) : rule("SQLPG283");
+    case "policy":
+      return created.has(bare(n.table.pieces.join("."))) ? rule("SQLPG200", { note: "a policy on a table this file creates" }) : rule("SQLPG290");
+    case "grant":
+      return n.defaults ? rule("SQLPG298") : n.revoke ? rule("SQLPG297") : rule("SQLPG296");
+    case "rowSecurity":
+      return rule("SQLPG293");
+    case "comment":
+      return rule("SQLPG216");
+    case "role":
+      return rule("SQLPG200");
+    default:
+      return rule("SQLPG200");
+  }
+}
+
+const DATA_WORDS = new Set(["INSERT", "UPDATE", "DELETE", "SELECT", "WITH", "COPY", "CALL", "DO", "SET", "BEGIN", "START", "COMMIT", "END", "ROLLBACK", "SAVEPOINT", "RELEASE", "ANALYZE", "ANALYSE", "VACUUM", "REINDEX", "LOCK", "NOTIFY", "LISTEN", "RESET", "SHOW", "EXPLAIN", "VALUES", "MERGE", "REFRESH", "CLUSTER", "CHECKPOINT", "DISCARD", "PREPARE", "EXECUTE", "DEALLOCATE"]);
+
+/** Classify one statement. `created` is the tables the file's earlier statements created. */
+function read(sql: string, created: Set<string>): Reading[] {
+  let tokens: Token[];
+  try {
+    tokens = tokenizeText(sql, 0);
+  } catch {
+    return [{ none: "text the tokenizer could not read" }];
+  }
+  const words = tokens.filter((t) => !isTrivia(t) && t.text !== ";").map((t) => t.text);
+  const u = words.map((w) => w.toUpperCase());
+  const head = u[0];
+  if (head === undefined) return [{ data: "an empty statement" }];
+
+  if (head === "CREATE" || head === "GRANT" || head === "REVOKE" || head === "COMMENT" || (head === "ALTER" && (u[1] === "DEFAULT" || (u[1] === "TABLE" && has(u, "ROW", "LEVEL", "SECURITY"))))) {
+    try {
+      const nodes = parseStatements(tokens);
+      return nodes.map((n) => fromNode(n, created));
+    } catch {
+      if (head === "CREATE") return [createByWords(u)];
+      if (head === "GRANT") return [rule("SQLPG296")];
+      if (head === "REVOKE") return [rule("SQLPG297")];
+      if (head === "COMMENT") return [rule("SQLPG216")];
+      if (head === "ALTER" && u[1] === "DEFAULT") return [rule("SQLPG298")];
+      return [rule("SQLPG293")];
+    }
+  }
+  if (head === "ALTER" && u[1] === "TABLE") {
+    let i = 2;
+    if (u[i] === "IF" && u[i + 1] === "EXISTS") i += 2;
+    if (u[i] === "ONLY") i++;
+    const { name, next } = nameAt(words, i);
+    if (u[next] === "*") i = next + 1;
+    else i = next;
+    const actions = splitTopLevel(words.slice(i));
+    if (actions.length === 0) return [{ none: `ALTER TABLE ${name} with no action` }];
+    void name;
+    return actions.map(tableAction);
+  }
+  if (head === "ALTER") return [alterOther(u, words)];
+  if (head === "DROP") return [dropStatement(u)];
+  if (head === "TRUNCATE") return [{ data: "TRUNCATE removes every row of the table" }];
+  if (DATA_WORDS.has(head)) return [{ data: `${head} reads or writes rows or the session, not the schema` }];
+  return [{ none: `${head} is not a statement the classifier reads` }];
+}
+
+/**
+ * Classify a migration's up statements, in order. The statements are the
+ * file's own text, one per entry (a Flyway file after placeholder
+ * substitution).
+ */
+export function classifyPgStatements(statements: readonly string[]): StatementVerdict[] {
+  const created = new Set<string>();
+  return statements.map((sql, k) => {
+    const readings = read(sql, created);
+    const out: StatementVerdict = { index: k + 1, sql: sqlHead(sql), outcome: "classified", changes: [] };
+    const none: string[] = [];
+    for (const r of readings) {
+      if ("rules" in r) out.changes.push(...r.rules.map((x) => classified(x.id, x.extra)));
+      else if ("undecided" in r) {
+        out.candidates = r.undecided.map((id) => classified(id));
+        out.note = r.note;
+      } else if ("none" in r) none.push(r.none);
+      else out.note = r.data;
+    }
+    if (out.candidates && out.changes.length === 0) out.outcome = "undecided";
+    else if (out.candidates) {
+      // A statement with some decided clauses and one undecided: report undecided, and keep the decided ones.
+      out.outcome = "undecided";
+    } else if (out.changes.length === 0 && none.length === 0) out.outcome = "data";
+    else if (none.length > 0) {
+      out.outcome = out.changes.length > 0 ? "classified" : "unclassified";
+      out.note = [out.note, `no rule covers ${none.join("; ")}`].filter(Boolean).join("; ");
+    }
+    return out;
+  });
+}

package/src/postgres/secret-options.ts

@@ -0,0 +1,10 @@
+/**
+ * Options that hold a credential. A foreign data wrapper's or server's
+ * `OPTIONS (...)` can carry any name, and a password in one would be written
+ * into a schema file by import and sent to every reader of the plan. The
+ * environment supplies credentials, in a user mapping (which is not declared)
+ * or its own secret store, as for a role's password (`./entities.ts`).
+ */
+
+/** Whether an option name reads as a credential: `password`, `sslpassword`, `secret_key`, `api_token`. */
+export const isSecretOption = (name: string): boolean => /pass(word|wd)?|secret|token|api[_-]?key|credential/i.test(name);

package/src/postgres/testing/roundtrip-schema.ts

@@ -5,7 +5,9 @@
  * sequence options, storage parameters, comments), an extension and the
  * domain over its type, a collation, composite and range types (and a
  * multirange column), plain and ordered-set aggregates, a cast and an event
- * trigger (#3911), a partitioned table and a partition, an ORM's
+ * trigger, an operator, extended statistics, a foreign data wrapper and server, and a
+ * publication (#3911), a foreign table, an operator class and family, a text
+ * search dictionary and configuration (#4406), a partitioned table and a partition, an ORM's
  * revision table that import leaves out, and an object carrying chant's
  * ownership trailer.
  */
@@ -60,5 +62,26 @@ CREATE CAST (app.pair AS text) WITH FUNCTION app.pair_text(app.pair) AS ASSIGNME
 COMMENT ON CAST (app.pair AS text) IS 'a:b';
 CREATE FUNCTION app.audit() RETURNS event_trigger LANGUAGE plpgsql AS $$ BEGIN END $$;
 CREATE EVENT TRIGGER app_audit ON ddl_command_end WHEN TAG IN ('CREATE TABLE') EXECUTE FUNCTION app.audit();
+CREATE FUNCTION app.int_same(a integer, b integer) RETURNS boolean LANGUAGE sql IMMUTABLE AS $$ SELECT a = b $$;
+CREATE OPERATOR app.=== (FUNCTION = app.int_same, LEFTARG = integer, RIGHTARG = integer, COMMUTATOR = OPERATOR(app.===), RESTRICT = eqsel, JOIN = eqjoinsel, HASHES, MERGES);
+COMMENT ON OPERATOR app.===(integer, integer) IS 'Same value';
+CREATE STATISTICS app.orders_stats (dependencies) ON user_id, status FROM app.orders;
+COMMENT ON STATISTICS app.orders_stats IS 'Users and their order states';
+CREATE FOREIGN DATA WRAPPER app_fdw OPTIONS (debug 'true');
+COMMENT ON FOREIGN DATA WRAPPER app_fdw IS 'Remote tables';
+CREATE SERVER app_replica TYPE 'pg' VERSION '16' FOREIGN DATA WRAPPER app_fdw OPTIONS (host 'db.internal', port '5432');
+COMMENT ON SERVER app_replica IS 'The read replica';
+CREATE PUBLICATION app_pub FOR TABLE app.orders, app.users WITH (publish = 'insert, update');
+COMMENT ON PUBLICATION app_pub IS 'Order changes';
+CREATE FOREIGN TABLE app.remote_orders (id integer OPTIONS (column_name 'order_id') NOT NULL, status text DEFAULT 'placed', CONSTRAINT status_known CHECK (status IS NOT NULL)) SERVER app_replica OPTIONS (table_name 'orders');
+COMMENT ON FOREIGN TABLE app.remote_orders IS 'Orders on the replica';
+CREATE OPERATOR CLASS app.int_hash_ops FOR TYPE integer USING hash AS OPERATOR 1 =, FUNCTION 1 hashint4(integer);
+COMMENT ON OPERATOR CLASS app.int_hash_ops USING hash IS 'Hash by value';
+CREATE OPERATOR FAMILY app.spare_ops USING btree;
+CREATE TEXT SEARCH DICTIONARY app.shop_stop (TEMPLATE = simple, StopWords = english);
+CREATE TEXT SEARCH CONFIGURATION app.shop (PARSER = pg_catalog."default");
+ALTER TEXT SEARCH CONFIGURATION app.shop ADD MAPPING FOR asciiword, word WITH app.shop_stop, english_stem;
+ALTER TEXT SEARCH CONFIGURATION app.shop ADD MAPPING FOR int, uint WITH simple;
+COMMENT ON TEXT SEARCH CONFIGURATION app.shop IS 'Shop search';
 CREATE TABLE public._prisma_migrations (id varchar(36) PRIMARY KEY, checksum varchar(64) NOT NULL);
 `;

package/src/postgres.ts

@@ -20,6 +20,16 @@ export {
   aggregate,
   trigger,
   eventTrigger,
+  operator,
+  operatorClass,
+  operatorFamily,
+  foreignTable,
+  textSearchConfiguration,
+  textSearchDictionary,
+  statistics,
+  foreignDataWrapper,
+  foreignServer,
+  publication,
   collation,
   cast,
   policy,
@@ -49,6 +59,16 @@ export {
   type PostgresCollation,
   type PostgresCast,
   type PostgresEventTrigger,
+  type PostgresOperator,
+  type PostgresOperatorClass,
+  type PostgresOperatorFamily,
+  type PostgresForeignTable,
+  type PostgresTextSearchConfiguration,
+  type PostgresTextSearchDictionary,
+  type PostgresStatistics,
+  type PostgresForeignDataWrapper,
+  type PostgresForeignServer,
+  type PostgresPublication,
   type PostgresExtension,
   type PostgresFunction,
   type PostgresProcedure,
@@ -75,6 +95,19 @@ export {
   type CollationProps,
   type CastProps,
   type EventTriggerProps,
+  type OperatorProps,
+  type OperatorClassProps,
+  type OperatorClassItemDef,
+  type OperatorFamilyProps,
+  type ForeignTableProps,
+  type TextSearchConfigurationProps,
+  type TextSearchDictionaryProps,
+  type TextSearchMappingDef,
+  type StatisticsProps,
+  type ForeignDataWrapperProps,
+  type ForeignServerProps,
+  type PublicationProps,
+  type PublicationTableDef,
   type DomainProps,
   type ExtensionProps,
   type RoutineProps,

package/src/serializer.ts

@@ -21,6 +21,11 @@
  * of each. Two databases of different dialects are two projects (two
  * workspace members, #3047 question 1).
  *
+ * A ClickHouse object that names a named collection (an engine's first
+ * argument, a table function's, a dictionary source's `NAME`) carries
+ * `namedCollections`, and the document lists every one the build names, so
+ * a plan can check each exists on the server (#4395).
+ *
  * A Postgres document also carries `postgresMajor`, the major the project
  * targets (`sql.postgresMajor`, else the newest pinned major). Post-synth
  * checks and the lock classifier read it; ClickHouse output has no such field.
@@ -43,6 +48,7 @@ import { sqlFileLineOf, sqlFileOf, sqlFileReferences } from "./sql-files";
 import { relative } from "node:path";
 import { getProvenance } from "@intentius/chant/provenance";
 import type { UnparsedClause } from "./core/interpolation";
+import { namedCollectionReferences, type CollectionReferrer } from "./clickhouse/named-collections";
 
 export { applyOrder } from "./core/references";
 
@@ -89,8 +95,9 @@ function objectJson(name: string, obj: ClickHouseObject, names: Map<Declarable,
     props.lineage = lineageJson(props.lineage as LineageEdge[], names);
   }
   const dependsOn = [...new Set([...obj.dependsOn.map((r) => referenceName(r, names)).filter((n) => n !== undefined), ...extra])];
+  const collections = namedCollectionReferences(props as CollectionReferrer);
   return JSON.parse(
-    JSON.stringify({ export: name, type: obj.entityType, sqlName: obj.sqlName, ...props, dependsOn, ddl }),
+    JSON.stringify({ export: name, type: obj.entityType, sqlName: obj.sqlName, ...props, dependsOn, ...(collections.length > 0 ? { namedCollections: collections } : {}), ddl }),
   ) as Record<string, unknown>;
 }
 
@@ -170,10 +177,14 @@ export const sqlSerializer: Serializer = {
 
     const extra = sqlFileReferences("clickhouse", objects);
     const order = applyOrder(objects, names, extra);
+    const json = order.map((n) => objectJson(n, objects.get(n)!, names, extra.get(n)));
+    // The named collections the build references (#4395), each once, by name: a plan checks each exists.
+    const collections = [...new Set(json.flatMap((o) => (o.namedCollections as string[] | undefined) ?? []))].sort();
     const doc = {
       dialect: "clickhouse",
       applyOrder: order,
-      objects: order.map((n) => objectJson(n, objects.get(n)!, names, extra.get(n))),
+      ...(collections.length > 0 ? { namedCollections: collections } : {}),
+      objects: json,
     };
     const ddl = order
       .map((n) => `${(objects.get(n)!.props as { ddl: string }).ddl};`)

package/src/skills/chant-sql-postgres-migration.md

@@ -114,13 +114,14 @@ or set `replicationLag: false` to run without the check.
 The Plan phase refuses, naming each object and what to do, a table or column with:
 
 - something on the column it does not carry over: an exclusion constraint, a constraint trigger, a materialized view, a rule, a trigger, a policy, a statistics object, a generated column computed from it, an INVALID index, or a view the build does not declare;
-- on a partitioned table, an index, key or foreign key on the column, or the column in the partition key (chant #3333);
+- on a partitioned table, a key or foreign key on the column, or the column in the partition key;
 - a partition (run the Op on its partitioned table) or an inheritance tree (chant #3333);
 - a generated column itself, which the applier changes in place;
 - no primary key;
-- a rename combined with a type change: run the Op twice, the rename first (chant #3331);
-- a rename of a column with a default, an identity or a serial sequence: drop the default, rename, declare it again (chant #3331);
+- a rename combined with a type change, unless the Op gives `reverse` (the old column's value from the new one); with a default, a serial or an identity as well, run the Op twice, the rename first;
+- a rename of a column with a default, an identity or a serial sequence, unless the Op gives `defaultAtContract: true` (the default lands on the new column at the contract; an explicit NULL into the new name gets the default until then);
 - a column a logical replication publication sends; a column list that leaves it out does not stop the Op (chant #3332);
+- a table that is the target of a logical replication subscription, named in the refusal (chant #4405);
 - a column that needs no migration, where the message says to use the applier.
 
 ## Checking it is done

package/src/skills/chant-sql-postgres.md

@@ -42,6 +42,16 @@ The export name (`users`) is the object's identity in chant. The name in the SQL
 | `aggregate` | `CREATE AGGREGATE` | `Postgres::Aggregate` |
 | `trigger` | `CREATE TRIGGER` | `Postgres::Trigger` |
 | `eventTrigger` | `CREATE EVENT TRIGGER` | `Postgres::EventTrigger` |
+| `operator` | `CREATE OPERATOR` | `Postgres::Operator` |
+| `operatorClass` | `CREATE OPERATOR CLASS` | `Postgres::OperatorClass` |
+| `operatorFamily` | `CREATE OPERATOR FAMILY` | `Postgres::OperatorFamily` |
+| `foreignTable` | `CREATE FOREIGN TABLE` | `Postgres::ForeignTable` |
+| `textSearchConfiguration` | `CREATE TEXT SEARCH CONFIGURATION`, then `ALTER TEXT SEARCH CONFIGURATION ... ADD MAPPING` | `Postgres::TextSearchConfiguration` |
+| `textSearchDictionary` | `CREATE TEXT SEARCH DICTIONARY` | `Postgres::TextSearchDictionary` |
+| `statistics` | `CREATE STATISTICS` | `Postgres::Statistics` |
+| `foreignDataWrapper` | `CREATE FOREIGN DATA WRAPPER` | `Postgres::ForeignDataWrapper` |
+| `foreignServer` | `CREATE SERVER` | `Postgres::ForeignServer` |
+| `publication` | `CREATE PUBLICATION` | `Postgres::Publication` |
 | `collation` | `CREATE COLLATION` | `Postgres::Collation` |
 | `cast` | `CREATE CAST` | `Postgres::Cast` |
 | `policy` | `CREATE POLICY` | `Postgres::Policy` |
@@ -189,4 +199,4 @@ chant build src --lexicon sql -o dist/schema.json
 
 ## Not covered
 
-`ALTER` (other than a table's row-level security), role memberships, rules, operators, extended statistics, foreign data objects, publications and subscriptions, text search objects and base types are not declared with the tags. An aggregate is identified by its argument types; a cast is named `(source AS target)` and an event trigger lives in no schema. A definition change to a composite or range type, collation, cast or event trigger is a drop and a create, refused while something depends on it. Planning a schema change (`chant sql diff`, `chant sql plan`, the lock classes) is the `chant-sql-postgres-plan` skill. Apply and the expand-and-contract migration Op have their own skills, which arrive with the code they describe.
+`ALTER` (other than a table's row-level security), role memberships, rules, operator classes and families, foreign tables, user mappings, subscriptions, text search objects and base types are not declared with the tags. Import names an object of such a kind and stops; `docs/pages/object-coverage.mdx` lists every `CREATE` command and why. An aggregate is identified by its argument types; a cast is named `(source AS target)` and an event trigger lives in no schema. A definition change to a composite or range type, collation, cast or event trigger is a drop and a create, refused while something depends on it. Planning a schema change (`chant sql diff`, `chant sql plan`, the lock classes) is the `chant-sql-postgres-plan` skill. Apply and the expand-and-contract migration Op have their own skills, which arrive with the code they describe.

package/src/skills/chant-sql.md

@@ -114,6 +114,7 @@ When chant interprets a composite's factory at build time, each field records th
 | SQLCH002 | lint | a `Nullable` column in the sort key or primary key, which ClickHouse refuses without `allow_nullable_key` |
 | SQLCH003 | lint | `${events.kind}` where `${events.columns.kind}` was meant |
 | SQLCH101 | build | an engine the pinned ClickHouse server does not have |
+| SQLCH128 | build | a literal credential in an integration engine's or table function's arguments; name a named collection instead |
 
 ## Commands
 
@@ -126,6 +127,6 @@ chant build src --lexicon sql -o dist/schema.json
 
 ## Not covered
 
-`CREATE TABLE ... AS` and `CREATE TABLE ... AS SELECT` are not declared with the tags. Dictionaries, functions, named collections and access control are not modelled. A second dialect is not supported.
+`CREATE TABLE ... AS` and `CREATE TABLE ... AS SELECT` are not declared with the tags. A named collection is never declared: name one in an integration engine, a table function or a dictionary `SOURCE` (`ENGINE = S3(lake_creds, format = 'Parquet')`), and `chant sql plan` checks that the server has it. A second dialect is not supported.
 
 Note that `chant migrate` does not run schema migrations. Schema changes go through `chant sql plan`, apply and the rebuild Op, described in the `chant-sql-plan` and `chant-sql-rebuild` skills.

package/src/sql-files.test.ts

@@ -42,6 +42,39 @@ describe("reading a file of Postgres DDL", () => {
     );
   });
 
+  test("a foreign table, an operator class and a text search configuration with its mappings and comments are each one object (#4406)", () => {
+    const ddl = [
+      "CREATE FOREIGN TABLE app.remote_orders (id int NOT NULL) SERVER replica OPTIONS (table_name 'orders');",
+      "COMMENT ON FOREIGN TABLE app.remote_orders IS 'Orders';",
+      "CREATE OPERATOR CLASS app.int_hash_ops FOR TYPE int USING hash AS OPERATOR 1 =, FUNCTION 1 hashint4(int);",
+      "CREATE OPERATOR FAMILY app.int_hash_ops USING btree;",
+      "COMMENT ON OPERATOR CLASS app.int_hash_ops USING hash IS 'Hash';",
+      "CREATE TEXT SEARCH DICTIONARY app.shop_stop (TEMPLATE = simple, StopWords = english);",
+      "CREATE TEXT SEARCH CONFIGURATION app.shop (PARSER = pg_catalog.default);",
+      "CREATE TEXT SEARCH CONFIGURATION app.shop2 (PARSER = pg_catalog.default);",
+      "ALTER TEXT SEARCH CONFIGURATION app.shop ADD MAPPING FOR word WITH app.shop_stop;",
+      "COMMENT ON TEXT SEARCH CONFIGURATION app.shop2 IS 'Second';",
+    ].join("\n");
+    const objects = readSqlFile("postgres", ddl, { origin: "schema.sql" });
+    expect(objects.map((o) => [o.export, o.type, o.name])).toEqual([
+      ["remoteOrders", "Postgres::ForeignTable", "app.remote_orders"],
+      ["appIntHashOps", "Postgres::OperatorClass", "app.int_hash_ops"],
+      ["appIntHashOps2", "Postgres::OperatorFamily", "app.int_hash_ops"],
+      ["shopStop", "Postgres::TextSearchDictionary", "app.shop_stop"],
+      ["shop", "Postgres::TextSearchConfiguration", "app.shop"],
+      ["shop2", "Postgres::TextSearchConfiguration", "app.shop2"],
+    ]);
+    expect(objects[0]!.ddl).toContain("COMMENT ON FOREIGN TABLE app.remote_orders IS 'Orders'");
+    // A class and a family of one name and two methods are two objects, and each takes only its own comment.
+    expect(objects[1]!.ddl).toContain("COMMENT ON OPERATOR CLASS app.int_hash_ops USING hash IS 'Hash'");
+    expect(objects[2]!.ddl).not.toContain("COMMENT");
+    expect(objects[4]!.ddl).toContain("ADD MAPPING FOR word WITH app.shop_stop");
+    expect(objects[5]!.ddl).toContain("COMMENT ON TEXT SEARCH CONFIGURATION app.shop2 IS 'Second'");
+    expect(objects[5]!.ddl).not.toContain("ADD MAPPING");
+    // A mapping for a configuration the file does not create is named.
+    expect(() => readSqlFile("postgres", "ALTER TEXT SEARCH CONFIGURATION app.x ADD MAPPING FOR word WITH simple;", { origin: "x.sql" })).toThrow(/ALTER TEXT SEARCH CONFIGURATION \.\.\. ADD MAPPING for an object the file does not create/);
+  });
+
   test("a statement that declares nothing or does not parse is an error naming it, and every one is named", () => {
     const read = () => readSqlFile("postgres", "CREATE TABLE a (id int);\nINSERT INTO a VALUES (1);\nCREATE TABLE b (id int,);\nCOMMENT ON TABLE c IS 'x';", { origin: "db/schema.sql" });
     expect(read).toThrow(SqlFileError);

package/src/sql-types-command.ts

@@ -18,6 +18,12 @@ import { basename, dirname, join, relative, resolve } from "node:path";
 import type { CommandGroupContext } from "@intentius/chant/cli/command-group";
 import { renderSqlTypes, SQL_TYPES_FILE } from "./sql-types";
 
+export interface ProjectLocation {
+  config: Awaited<ReturnType<typeof import("@intentius/chant/config").loadChantConfigUpward>>["config"];
+  projectDir: string;
+  sourceDir: string;
+}
+
 const USAGE = "usage: chant sql types [--output <path>] [--check]";
 
 /** The `--output` value, or an error message. */
@@ -43,37 +49,53 @@ export function shadowingSource(path: string): string | undefined {
   return existsSync(source) ? source : undefined;
 }
 
-export async function runTypes(ctx: CommandGroupContext): Promise<number> {
-  const args = parseArgs(ctx.rawArgs);
-  if ("error" in args) {
-    console.error(`${args.error}\n${USAGE}`);
-    return 1;
-  }
-  const cwd = process.cwd();
+/** The project the working directory belongs to: its config, its root and its source directory. */
+export async function loadProject(cwd: string): Promise<ProjectLocation> {
   const { loadChantConfigUpward } = await import("@intentius/chant/config");
   const { config, configPath } = await loadChantConfigUpward(cwd);
   const projectDir = configPath ? dirname(configPath) : cwd;
-  const sourceDir = resolve(projectDir, config.sourceDir ?? ".");
-  const output = args.output !== undefined ? resolve(cwd, args.output) : join(sourceDir, SQL_TYPES_FILE);
-  const shadow = shadowingSource(output);
-  if (shadow) {
-    console.error(`chant sql types: ${relative(cwd, shadow) || basename(shadow)} sits beside ${relative(cwd, output) || basename(output)}, and tsc reads it instead of the declarations; rename one`);
-    return 1;
-  }
+  return { config, projectDir, sourceDir: resolve(projectDir, config.sourceDir ?? ".") };
+}
 
+/**
+ * The fold build's entities: what `chant build` declares, `.sql` files and
+ * composite members included. Prints the build's errors and returns
+ * undefined when it fails.
+ */
+export async function buildFolded(project: ProjectLocation, command: string, consequence: string) {
+  const { config, projectDir, sourceDir } = project;
   const { build } = await import("@intentius/chant/build");
   const { resolveBuildModes, resolveProjectBuildOptions } = await import("@intentius/chant/cli/build-options");
   const { loadPlugins, resolveProjectLexicons } = await import("@intentius/chant/cli/plugins");
   const { sqlSerializer } = await import("./serializer");
   const plugins = await loadPlugins(await resolveProjectLexicons(projectDir, { lexicon: "sql" }));
-  // The fold build's entities: what `chant build` declares, `.sql` files and composite members included.
   const options = resolveProjectBuildOptions({ config, configDir: projectDir, plugins, modes: resolveBuildModes(config, { fold: true }) });
   const result = await build(sourceDir, [sqlSerializer], undefined, options);
   if (result.errors.length > 0) {
     for (const e of result.errors) console.error(e instanceof Error ? e.message : String(e));
-    console.error("chant sql types: the build failed, so nothing was written");
+    console.error(`${command}: the build failed, so ${consequence}`);
+    return undefined;
+  }
+  return result;
+}
+
+export async function runTypes(ctx: CommandGroupContext): Promise<number> {
+  const args = parseArgs(ctx.rawArgs);
+  if ("error" in args) {
+    console.error(`${args.error}\n${USAGE}`);
     return 1;
   }
+  const cwd = process.cwd();
+  const { config, projectDir, sourceDir } = await loadProject(cwd);
+  const output = args.output !== undefined ? resolve(cwd, args.output) : join(sourceDir, SQL_TYPES_FILE);
+  const shadow = shadowingSource(output);
+  if (shadow) {
+    console.error(`chant sql types: ${relative(cwd, shadow) || basename(shadow)} sits beside ${relative(cwd, output) || basename(output)}, and tsc reads it instead of the declarations; rename one`);
+    return 1;
+  }
+
+  const result = await buildFolded({ config, projectDir, sourceDir }, "chant sql types", "nothing was written");
+  if (!result) return 1;
   const text = renderSqlTypes(result.entities, { projectDir, config: config as unknown as Record<string, unknown> });
   const shown = relative(cwd, output);
   const named = shown && !shown.startsWith("..") ? shown : output;