@intentius/chant-lexicon-sql 0.0.0-stage → 0.112.1

package/src/skill-defs.test.ts

@@ -0,0 +1,81 @@
+import { describe, expect, test } from "vitest";
+import { readFileSync } from "node:fs";
+import { sqlPlugin } from "./plugin";
+
+describe("skills", () => {
+  const skills = sqlPlugin.skills!();
+
+  test("six skills, each with its file's content and a matching frontmatter name", () => {
+    expect(skills.map((s) => s.name)).toEqual(["chant-sql", "chant-sql-plan", "chant-sql-rebuild", "chant-sql-postgres", "chant-sql-postgres-plan", "chant-sql-postgres-migration"]);
+    for (const s of skills) {
+      expect(s.content.length, s.name).toBeGreaterThan(500);
+      expect(s.content).toContain(`skill: ${s.name}\n`);
+    }
+  });
+
+  test("the rebuild skill declares the shipped Op", () => {
+    const rebuild = skills.find((s) => s.name === "chant-sql-rebuild")!;
+    expect(rebuild.content).toContain('import { ClickHouseRebuildOp } from "@intentius/chant-lexicon-sql/clickhouse"');
+  });
+
+  test("the Postgres skill cites only SQLPG ids that exist, and the ClickHouse skills none", async () => {
+    const { rules } = await import("./lint/rules");
+    const known = new Set<string>(rules.map((r) => r.id));
+    const pg = skills.find((s) => s.name === "chant-sql-postgres")!;
+    const ids = pg.content.match(/SQLPG\d{3}/g) ?? [];
+    expect(ids.length).toBeGreaterThan(0);
+    for (const id of ids) expect(known.has(id), id).toBe(true);
+    expect(pg.content).not.toMatch(/SQLCH\d{3}/);
+    for (const s of skills.filter((s) => !s.name.startsWith("chant-sql-postgres"))) {
+      if (["chant-sql", "chant-sql-plan", "chant-sql-rebuild"].includes(s.name)) expect(s.content, s.name).not.toMatch(/Postgres|SQLPG/);
+    }
+  });
+
+  test("the Postgres plan skill cites only classifier rules that exist, names no ClickHouse rule, and stays Postgres-only", async () => {
+    const { PG_CLASSIFIER_RULES } = await import("./postgres/plan/rules");
+    const plan = skills.find((s) => s.name === "chant-sql-postgres-plan")!;
+    const ids = [...new Set(plan.content.match(/SQLPG2\d\d/g) ?? [])];
+    expect(ids.length).toBeGreaterThan(20);
+    for (const id of ids) expect(id in PG_CLASSIFIER_RULES, id).toBe(true);
+    expect(plan.content).not.toMatch(/SQLCH|ClickHouse|MergeTree/);
+  });
+
+  test("every SQLPG0xx and SQLPG1xx id the Postgres skills cite is a lint rule or a post-synth check", async () => {
+    const { rules } = await import("./lint/rules");
+    const { postSynthChecks } = await import("./lint/post-synth");
+    const known = new Set<string>([...rules.map((r) => r.id), ...postSynthChecks.map((c) => c.id)]);
+    for (const s of skills.filter((s) => s.name.startsWith("chant-sql-postgres"))) {
+      for (const id of s.content.match(/SQLPG[01]\d\d/g) ?? []) expect(known.has(id), `${s.name}: ${id}`).toBe(true);
+    }
+  });
+
+  test("every SQLCH id a skill cites exists in the lexicon", async () => {
+    const { rules } = await import("./lint/rules");
+    const { postSynthChecks } = await import("./lint/post-synth");
+    const known = new Set<string>([...rules.map((r) => r.id), ...postSynthChecks.map((c) => c.id)]);
+    for (const s of skills) {
+      // The SQLCH2xx ids are the classifier's, which come from the plan rules, not from a lint rule.
+      for (const id of s.content.match(/SQLCH0\d\d|SQLCH1\d\d/g) ?? []) expect(known.has(id), `${s.name}: ${id}`).toBe(true);
+    }
+  });
+
+  test("the Postgres migration skill declares the shipped Op with options it has, and cites only rules that exist", async () => {
+    const { PG_CLASSIFIER_RULES } = await import("./postgres/plan/rules");
+    const src = readFileSync(new URL("./postgres/migrate/op.ts", import.meta.url), "utf8");
+    const iface = /export interface PostgresMigrationOpConfig \{([\s\S]*?)\n\}/.exec(src)![1]!;
+    const options = new Set([...iface.matchAll(/^  (\w+)\??:/gm)].map((m) => m[1]!));
+    const mig = skills.find((s) => s.name === "chant-sql-postgres-migration")!;
+    expect(mig.content).toContain('import { PostgresMigrationOp } from "@intentius/chant-lexicon-sql/postgres"');
+    // Every option named in the options list, and every key of a declaration, is on the Op's config.
+    const listed = [...mig.content.matchAll(/^- `(\w+)`(?: \(| and `(\w+)`|:)/gm)].flatMap((m) => [m[1], m[2]]).filter((x): x is string => !!x);
+    const declared = [...mig.content.matchAll(/PostgresMigrationOp\(\{([^}]*)\}\)/g), ...mig.content.matchAll(/PostgresMigrationOp\(\{([\s\S]*?)\n\}\)/g)]
+      .flatMap((m) => [...m[1]!.replace(/\/\/.*$/gm, "").matchAll(/(?:^|[\s,{])(\w+):/g)].map((k) => k[1]!));
+    expect(listed.length).toBeGreaterThan(5);
+    for (const o of [...listed, ...declared]) expect(options.has(o), o).toBe(true);
+    for (const o of ["env", "table", "column", "using", "batchSize", "retain", "replicationLag", "gate", "contractGate"]) expect(mig.content, o).toContain(`\`${o}\``);
+    const ids = [...new Set(mig.content.match(/SQLPG\d{3}/g) ?? [])];
+    expect(ids.length).toBeGreaterThan(1);
+    for (const id of ids) expect(id in PG_CLASSIFIER_RULES, id).toBe(true);
+    expect(mig.content).not.toMatch(/SQLCH|ClickHouse|MergeTree/);
+  });
+});

package/src/skill-defs.ts

@@ -0,0 +1,102 @@
+import { createSkillsLoader } from "@intentius/chant/lexicon-plugin-helpers";
+
+/** The sql lexicon's AI skills, read from src/skills/. */
+export const sqlSkills = createSkillsLoader(import.meta.url, [
+  {
+    file: "chant-sql.md",
+    name: "chant-sql",
+    description:
+      "Declare ClickHouse databases, tables, views and materialized views as SQL-shaped tagged templates, with references, lineage and the checks that run on them",
+    triggers: [
+      { type: "context" as const, value: "clickhouse" },
+      { type: "context" as const, value: "clickhouse schema" },
+      { type: "context" as const, value: "materialized view" },
+    ],
+    examples: [
+      {
+        title: "A table and a view that reads it",
+        output:
+          'export const analytics = database`CREATE DATABASE analytics ENGINE = Atomic`;\n' +
+          "export const events = table`CREATE TABLE ${analytics}.events (user_id UUID, ts DateTime) ENGINE = MergeTree ORDER BY (user_id, ts)`;\n" +
+          "export const recent = view`CREATE VIEW ${analytics}.recent AS SELECT ${events.columns.user_id} AS user_id FROM ${events}`;",
+      },
+    ],
+  },
+  {
+    file: "chant-sql-plan.md",
+    name: "chant-sql-plan",
+    description:
+      "Plan ClickHouse schema changes with chant sql diff and chant sql plan, and read the three change classes (metadata only, background rewrite, rebuild) with the ALTER restriction behind each",
+    triggers: [
+      { type: "context" as const, value: "clickhouse schema change" },
+      { type: "context" as const, value: "sql plan" },
+      { type: "context" as const, value: "alter table" },
+    ],
+    examples: [
+      {
+        title: "Classify a pull request's schema change offline",
+        output: "chant sql diff base.json head.json",
+      },
+    ],
+  },
+  {
+    file: "chant-sql-rebuild.md",
+    name: "chant-sql-rebuild",
+    description:
+      "Run a ClickHouse table rebuild (a sorting key, primary key, partition key, engine or key column type change) with ClickHouseRebuildOp, a gated Op with backfill receipts, verification, an exchange swap and onFailure cleanup",
+    triggers: [
+      { type: "context" as const, value: "clickhouse rebuild" },
+      { type: "context" as const, value: "change sorting key" },
+      { type: "context" as const, value: "clickhouse migration" },
+    ],
+  },
+  {
+    file: "chant-sql-postgres.md",
+    name: "chant-sql-postgres",
+    description:
+      "Declare Postgres schemas, tables, constraints, indexes, views, sequences, enum and domain types and extensions as SQL-shaped tagged templates, with references, lineage and the SQLPG lint rules",
+    triggers: [
+      { type: "context" as const, value: "postgres" },
+      { type: "context" as const, value: "postgres schema" },
+      { type: "context" as const, value: "postgresql" },
+    ],
+    examples: [
+      {
+        title: "A table with a foreign key and an index",
+        output:
+          'export const app = schema`CREATE SCHEMA app`;\n' +
+          "export const users = table`CREATE TABLE ${app}.users (id bigint PRIMARY KEY)`;\n" +
+          "export const orders = table`CREATE TABLE ${app}.orders (id bigint PRIMARY KEY, user_id bigint NOT NULL REFERENCES ${users} (${users.columns.id}))`;\n" +
+          "export const ordersUser = index`CREATE INDEX orders_user_id_idx ON ${orders} (${orders.columns.user_id})`;",
+      },
+    ],
+  },
+  {
+    file: "chant-sql-postgres-plan.md",
+    name: "chant-sql-postgres-plan",
+    description:
+      "Plan Postgres schema changes with chant sql diff and chant sql plan, and read the change classes (metadata only, validates, needs CONCURRENTLY, ACCESS EXCLUSIVE rewrite, expand and contract) as the lock each takes in production",
+    triggers: [
+      { type: "context" as const, value: "postgres schema change" },
+      { type: "context" as const, value: "postgres lock" },
+      { type: "context" as const, value: "postgres migration" },
+    ],
+    examples: [
+      {
+        title: "Classify a pull request's Postgres schema change offline",
+        output: "chant sql diff base.json head.json",
+      },
+    ],
+  },
+  {
+    file: "chant-sql-postgres-migration.md",
+    name: "chant-sql-postgres-migration",
+    description:
+      "Run a Postgres column rename or type change with PostgresMigrationOp, a gated expand-and-contract Op with a dual-write trigger, a batched backfill with receipts, verification, a switch, replication lag handling and onFailure cleanup",
+    triggers: [
+      { type: "context" as const, value: "postgres column rename" },
+      { type: "context" as const, value: "postgres expand and contract" },
+      { type: "context" as const, value: "postgres backfill" },
+    ],
+  },
+]);

package/src/skills/chant-sql-plan.md

@@ -0,0 +1,123 @@
+---
+skill: chant-sql-plan
+description: Plan ClickHouse schema changes with chant sql diff and chant sql plan, and read the three change classes (metadata only, background rewrite, rebuild) with the ALTER restriction behind each
+user-invocable: true
+---
+# Planning and applying ClickHouse schema changes
+
+Use this skill when a declared ClickHouse schema changes and you need to know what the change does to the server before anything runs. This skill covers what exists today: the offline diff, the plan against a live server, and the change classifier. The applier and the rebuild are described at the end with their current state.
+
+## Two commands
+
+```bash
+chant sql diff base.json head.json     # two chant build outputs, offline
+chant sql plan prod dist/schema.json   # a build output against the prod server
+```
+
+`chant sql diff` compares a pull request's base and head builds and needs no server. `chant sql plan <env>` reads the server that `sql.profiles.<env>` binds in `chant.config.ts`, and compares the build with it. Add `--json` to either for a machine-readable report.
+
+Both exit 2 when any change needs a rebuild, and 0 otherwise, so either can gate a pipeline.
+
+```ts
+// chant.config.ts
+import type { ChantConfig } from "@intentius/chant/config";
+import "@intentius/chant-lexicon-sql";
+
+export default {
+  lexicons: ["sql"],
+  sql: {
+    profiles: {
+      prod: {
+        url: "https://clickhouse.example.com:8443",
+        user: { env: "CH_USER" },
+        password: { env: "CH_PROD_PASSWORD" },
+        databases: ["analytics"],
+      },
+    },
+  },
+} satisfies ChantConfig;
+```
+
+Credentials are named by environment variable and never written into the config.
+
+## The three change classes
+
+| Class | Means | Examples |
+|---|---|---|
+| metadata only | the server records the change and touches no existing data | add a column (SQLCH201), change a default (SQLCH207), add a skip index (SQLCH204), rename a column outside the keys (SQLCH212), change a materialized view's query (SQLCH241) |
+| background rewrite | a mutation rewrites existing parts in the background, with no rollback | change a TTL (SQLCH205), change the type of a column outside the primary key (SQLCH210) |
+| rebuild | ClickHouse cannot make the change to the existing table; the data has to be copied into a new one | change the sorting key beyond appending new columns (SQLCH220), the partition key (SQLCH222), the engine or its arguments (SQLCH223), a materialized view's `TO` target (SQLCH242) |
+
+Every rule has an id in the SQLCH2xx range, names its class, and cites the ClickHouse documentation page that states the restriction. The full table is on the "Planning and the Change Classifier" page of the sql docs, written from the rules, so read the page rather than recalling a class from memory.
+
+Two classes of change are not what they look like:
+
+- A metadata-only drop (SQLCH202) is not reversible: the column's data is removed from every part.
+- A TTL change (SQLCH205) looks like metadata, but with `materialize_ttl_after_modify` on (the default) the server recalculates the TTL over existing data as a mutation.
+
+`chant lifecycle plan` reports the same classification as each update's disruption: metadata only is `in-place`, a rewrite is `rolling`, a rebuild is `replace`. For a column type or a sorting key change the lifecycle plan reports `unknown`, since deciding needs the whole definition; `chant sql plan` classifies those.
+
+## Identity and renames
+
+Between two builds an object is its export name, so a changed SQL name under the same export is a rename (`RENAME TABLE`, metadata only on an Atomic database). Against a server, which has no export names, an object is `database.name`. To rename something the server already has, put a comment before its CREATE:
+
+```sql
+-- previously: events_raw
+CREATE TABLE analytics.events (...)
+```
+
+A column is its name. Declare a rename on the column's own line:
+
+```sql
+event_kind LowCardinality(String), -- previously: kind
+```
+
+Without the hint, a rename plans as a drop and an add, and the report says so when the two have the same type and position. A drop is destructive, so add the hint.
+
+## No false drift
+
+The server rewrites DDL on `SHOW CREATE` (quoting, `INTERVAL 1 DAY` as `toIntervalDay(1)`, default codec levels, settings at their default). The plan undoes those rewrites with normalization rules, and asks the server's own formatter about any expression the rules leave different, so formatting alone is not reported as a change. `chant lifecycle diff <env> --live` reads property-level drift with the same normalization.
+
+## Reading a plan in a pull request
+
+1. `chant build src --lexicon sql -o head.json` on the pull request branch, and the same on the base branch into `base.json`.
+2. `chant sql diff base.json head.json`.
+3. A rebuild exits 2. Do not merge it as an ordinary change: the report names the `ClickHouseRebuildOp` declaration to add for each refused table (`rebuildOps` in `--json`), and the rebuild runs as that Op (`chant-sql-rebuild`).
+
+## Using plans in Ops
+
+The sql lexicon does not yet export step builders for these commands. Until it does, an Op can run them with `shell` and gate on the result:
+
+```ts
+import { Op, phase, build, shell, gate } from "@intentius/chant/op";
+
+export default Op({
+  name: "schema-plan",
+  overview: "Build the schema, plan it against prod, stop for approval",
+  phases: [
+    phase("Plan", [
+      build("src"),
+      shell("chant sql plan prod dist/schema.json --json", { json: true }),
+    ]),
+    phase("Approve", [gate("approve-schema-plan")]),
+  ],
+});
+```
+
+Resolve the gate with `chant approve approve-schema-plan`. The MCP server's `op-run`, `op-status` and `op-signal` tools start and watch the same Op.
+
+## Applying
+
+`ApplyOp` with `target: "clickhouse"` applies a build (`dist/schema.json`) to the server `sql.profiles.<env>` binds, through the sql lexicon's `clickhouseApply`. It makes the metadata-only and background-rewrite changes with `ALTER`, waits on `system.mutations` for a rewrite, and stamps chant's ownership marker on each object's comment. It reports a rebuild as not attempted (`unsupported-kind`, with the rule and the ALTER restriction in the detail) and sends nothing for that object, and withholds a column drop unless the Op may delete (`delete: "owned-only"` or `"gated"`). Prune drops only objects whose comment carries the project's marker.
+
+```typescript
+import { ApplyOp } from "@intentius/chant/op";
+
+const { op } = ApplyOp({ name: "schema-apply", env: "prod", target: "clickhouse", delete: "gated" });
+
+export default op;
+```
+
+Read the run's not-attempted count before calling a schema converged. A rebuild is never applied in place; it runs as the rebuild migration Op.
+
+`chant migrate` is not a schema migration runner. Schema changes go through plan, apply and the rebuild Op.

package/src/skills/chant-sql-postgres-migration.md

@@ -0,0 +1,126 @@
+---
+skill: chant-sql-postgres-migration
+description: Run a Postgres column rename or type change with PostgresMigrationOp, a gated expand-and-contract Op with a dual-write trigger, a batched backfill with receipts, verification, a switch, replication lag handling and onFailure cleanup
+user-invocable: true
+---
+# Running a Postgres column migration
+
+Use this skill when `chant sql plan` or `chant sql diff` exits 2 because a column's change is a rename (SQLPG205) or a type change across kinds (SQLPG208, text to integer). Postgres has no way to make these in place without breaking readers or failing on existing rows, and the applier refuses them. `PostgresMigrationOp` makes them as expand and contract: a new column is added beside the old, kept written, filled in batches, verified, and swapped in behind an approval.
+
+A type change within a kind that rewrites the table under `ACCESS EXCLUSIVE` (SQLPG207, integer to bigint) can run as the same Op: it fills the new column in batches instead of holding the lock for a rewrite. The plan does not hand that one over; write the declaration yourself. The other expand-and-contract rules (a NOT NULL column with no default, partitioning, an object rename, a view's columns, enum labels) have no Op.
+
+## Declare the Op
+
+The plan prints a declaration for each refused column, and `--json` carries them as `migrationOps`. Copy it into an `*.op.ts` file in the same pull request as the schema change:
+
+```typescript
+// ops/migrate-app-users-login.op.ts
+import { PostgresMigrationOp } from "@intentius/chant-lexicon-sql/postgres";
+
+export const { op } = PostgresMigrationOp({
+  name: "migrate-app-users-login",
+  env: "prod",                // sql.profiles.prod
+  table: "app.users",         // schema.name as the server knows it
+  column: "login",            // the column as the build declares it: the new name for a rename
+});
+```
+
+For a type change, `using` is the expression that computes the new value from the old row, as `ALTER COLUMN ... TYPE ... USING` takes it. It defaults to a cast to the declared type; set it when the cast is not the conversion you want:
+
+```typescript
+PostgresMigrationOp({ name: "migrate-orders-amount", env: "prod", table: "shop.orders", column: "amount", using: "NULLIF(amount, 'n/a')::numeric(12,2)" });
+```
+
+Options besides `name`, `env`, `table`, `column`, `using`:
+
+- `batchSize` (default 1000): rows per batch, the width of a range of a primary key of one integer column, or how many keys lie between two recorded boundaries of any other key;
+- `retain` (default `7d`): how long the old column is kept after the switch;
+- `replicationLag` (default `{ max: "10s", wait: "30m" }`, or `false`): see below;
+- `lockTimeoutMs` (default the profile's, else 5000) and `statementTimeoutMs` (default the profile's, else 60000, not applied to the scans of validation and verification);
+- `gate` and `contractGate`: the two approvals, each with `gate` (name), `timeout` and `description`; `gate` also takes `approval` for quorum, roles and a policy, and a policy gets `verifiedRows` and `mismatched` as context;
+- `backfillTimeout` (default `6h`), `output` (default `dist/schema.json`), `path`, `build` (default true runs `chant build` first), `stack`, `ownershipEnv`.
+
+## What it does
+
+| Phase | Step |
+|---|---|
+| Build | `chant build` |
+| Plan | classifies the column's change against the server and refuses what the Op does not make |
+| Expand | adds the new column, nullable, no default (a catalog change) |
+| Dual write | a trigger keeps the new column written: computed from the old one for a type change, both ways for a rename |
+| Backfill | `UPDATE` per primary-key range, each batch in its own transaction with its receipt, under `lock_timeout` |
+| Carry over | builds each index on the old column again on the new one `CONCURRENTLY` (a key's too), and adds each check and foreign key (another table's that references the column too) `NOT VALID`, then validates it |
+| Verify | compares rows, mismatches, NULLs and checksums of the new column against the old |
+| Approve | the switch gate |
+| Switch | proves NOT NULL (declared, or for a primary key or identity) with a `NOT VALID` check then `VALIDATE`; then one short transaction: a type change swaps the columns by name, a rename finishes the new column and keeps both written; the carried indexes and constraints take their names, keys with `ADD CONSTRAINT ... USING INDEX`; the views that read the column are made again from their declarations; an identity or serial sequence moves to the new column |
+| Retain | keeps the old column until `retain` has passed |
+| Approve contract | the contract gate |
+| Contract | drops the old column, and a rename's trigger |
+
+## Run it
+
+```bash
+chant run migrate-app-users-login        # expand, dual write, backfill, verify; exits 3 at approve-migrate-app-users-login
+chant run status migrate-app-users-login # Filled, Skipped, BackfilledRows, Verification, VerifiedRows
+chant approve migrate-app-users-login approve-migrate-app-users-login --actor <you>
+chant run migrate-app-users-login        # switches; exits 3 at approve-migrate-app-users-login-contract
+chant approve migrate-app-users-login approve-migrate-app-users-login-contract --actor <you>
+chant run migrate-app-users-login        # drops the old column once retain has passed
+```
+
+Each run goes as far as the next gate. Every step reads the server again, so a run after a gate, a crash or an approval carries on where the last stopped.
+
+The switch gate is bound to a digest of the plan and the verified new column. If the next run verifies different numbers, the gate asks again. For a rename, readers move to the new name at the switch, and the old name keeps working for writers until the contract, so deploy readers before approving. The contract gate is bound to the old column's identity and retention date; approving it early is fine, nothing is dropped before the date.
+
+## What it carries over
+
+What uses the column moves with it, so `integer` to `bigint` on a primary key that other tables reference, or a rename of a uniquely indexed `email`, needs nothing dropped first:
+
+- indexes, primary keys and unique constraints on the column are built on the new column `CONCURRENTLY` while writes go on;
+- checks and foreign keys, this table's and other tables' that reference the column, are added `NOT VALID` and validated;
+- views that read the column (and views on those) are dropped and created again from their declarations in the switch, with their grants and owner, so each such view must be declared in the build;
+- an identity moves to the new column under its sequence's name and carries on from the old sequence's position; a serial column's sequence is handed over and widened with it.
+
+A type change keeps every name. A rename takes the name the build declares for a key, foreign key or index on the new column, else turns a name Postgres made from the old column (`users_email_key`) into the one it makes from the new (`users_login_key`), and keeps the old column's unique constraints and indexes, renamed `__chant_old`, for readers of the old name until the contract.
+
+A partitioned table is migrated as one; its partitions follow it.
+
+## Receipts and resuming
+
+Each batch commits its `UPDATE` and its receipt in one transaction, in `<schema>.__chant_receipts` in the migrated table's schema. A primary key of one integer column is batched by ranges of its value; any other key (uuid, text, several columns) by boundaries walked from its index on the first run and recorded beside the receipts, so every run finds the same batches. A run killed during the backfill (Ctrl-C, a lost machine) is not a failure and runs no onFailure. The next run skips the batches that have receipts and fills the rest, so no batch is updated twice. The receipts bind the table's oid and the new column, so receipts from another table or an earlier attempt are not reused.
+
+## Replication lag
+
+Before each batch the backfill reads `pg_stat_replication` and pauses while a replica's replay lag is above `replicationLag.max`, for up to `wait` at a time. With no replicas it carries on. If `wait` passes with a replica still behind, the backfill stops with a message naming the replica, keeps its receipts, and the next run goes on from there.
+
+A role that is neither superuser nor a member of `pg_monitor` cannot read other sessions' lag, and the backfill stops saying so rather than guess. Grant it and run again:
+
+```sql
+GRANT pg_monitor TO migration_role;
+```
+
+or set `replicationLag: false` to run without the check.
+
+## When something goes wrong
+
+- A failed step (a value the `using` expression cannot convert, a verification mismatch, a refused declaration, an index that cannot be built) runs onFailure: the trigger, its function, the check, the carried indexes and constraints, the new column and the receipts table are dropped, and the plan shows the original change again. The old column was never touched. Fix the declaration or the data and run again. A failed gate or an aborted run does not run onFailure.
+- A lock timeout means another session holds a lock the step needs (SQLSTATE 55P03). The step retries a few times, then fails. Run again when the blocker has finished.
+- The working objects (new column, trigger, function, check, receipts table) carry chant's comment marker with a `migration=` key, and plans, imports and prunes leave them out, so an `ApplyOp` for the rest of the schema can run beside a migration. An object under one of the working names that is not this migration's stops the run; rename or drop it by hand.
+
+## What the Op refuses today
+
+The Plan phase refuses, naming each object and what to do, a table or column with:
+
+- something on the column it does not carry over: an exclusion constraint, a constraint trigger, a materialized view, a rule, a trigger, a policy, a statistics object, a generated column computed from it, an INVALID index, or a view the build does not declare;
+- on a partitioned table, an index, key or foreign key on the column, or the column in the partition key (chant #3333);
+- a partition (run the Op on its partitioned table) or an inheritance tree (chant #3333);
+- a generated column itself, which the applier changes in place;
+- no primary key;
+- a rename combined with a type change: run the Op twice, the rename first (chant #3331);
+- a rename of a column with a default, an identity or a serial sequence: drop the default, rename, declare it again (chant #3331);
+- a column a logical replication publication sends; a column list that leaves it out does not stop the Op (chant #3332);
+- a column that needs no migration, where the message says to use the applier.
+
+## Checking it is done
+
+`chant sql plan <env> dist/schema.json` reports no changes for the column, the old column and the receipts table are gone, and a later `chant run` of the Op reports done. Then remove the `*.op.ts` file.

package/src/skills/chant-sql-postgres-plan.md

@@ -0,0 +1,194 @@
+---
+skill: chant-sql-postgres-plan
+description: Plan Postgres schema changes with chant sql diff and chant sql plan, and read the change classes (metadata only, validates, needs CONCURRENTLY, ACCESS EXCLUSIVE rewrite, expand and contract) as the lock each takes in production
+user-invocable: true
+---
+# Planning Postgres schema changes
+
+Use this skill when a declared Postgres schema changes and you need to know what the change does to a running server before anything runs. It covers the offline diff, the plan against a live server, the classifier behind both, and applying a plan with the Postgres applier. The expand-and-contract migration Op has its own skill, `chant-sql-postgres-migration`.
+
+The classes are Postgres's own. A change is classified by the lock it takes and whether it reads or rewrites the table, so the class answers the production question: will this queue behind running queries, block writes, or fail?
+
+## Two commands
+
+```bash
+chant sql diff base.json head.json     # two chant build outputs, offline
+chant sql plan prod dist/schema.json   # a build output against the prod server
+```
+
+`chant sql diff` compares a pull request's base and head builds and needs no server. `chant sql plan <env>` reads the server that `sql.profiles.<env>` binds, and compares the build with it. Add `--json` to either for a machine-readable report. The build output's `dialect` decides which classifier runs; a Postgres build gets the one described here.
+
+Both exit 2 when any change can only be made as expand and contract, and 0 otherwise, so either can gate a pipeline. A plan refuses such a change: it is never one statement.
+
+```ts
+// chant.config.ts
+import type { ChantConfig } from "@intentius/chant/config";
+import "@intentius/chant-lexicon-sql";
+
+export default {
+  lexicons: ["sql"],
+  sql: {
+    dialect: "postgres",
+    postgresMajor: 16,
+    provider: "rds",
+    profiles: {
+      prod: {
+        url: "postgres://db.internal:5432/shop",
+        user: { env: "PGUSER" },
+        password: { env: "PG_PROD_PASSWORD" },
+        schemas: ["app"],
+        defaultSchema: "app",
+      },
+    },
+  },
+} satisfies ChantConfig;
+```
+
+Credentials are named by environment variable and never written into the URL. `schemas` limits what the plan reads to those schemas, and `defaultSchema` is where an unqualified declaration lives (`public` when omitted). Without a profile, `POSTGRES_URL`, `POSTGRES_USER` and `POSTGRES_PASSWORD` bind.
+
+## sql.postgresMajor
+
+`sql.postgresMajor` is the major the project targets (14 to 18); without it the newest supported major is used. The plan reads it, so set it to the server's major. It changes a classification where the majors differ:
+
+- changing a STORED generated column's expression (SQLPG212) is a rewrite from 17 on; on 14 to 16 there is no `SET EXPRESSION`, so the column is dropped and added and the change is expand and contract;
+- changing a table's access method (SQLPG226) is a rewrite from 15 on; on 14 it is expand and contract.
+
+The post-synth checks SQLPG114 to SQLPG116 read the same setting.
+
+## The change classes
+
+| Class | Report label | What it means in production |
+|---|---|---|
+| create | create | a new object; nothing existing changes |
+| metadata | metadata only | a catalog change; no row is read or written |
+| validate | validates under a weaker lock | every row is read to check a constraint, but reads and writes go on |
+| concurrently | needs CONCURRENTLY | an index build or drop that blocks writes unless it is done CONCURRENTLY |
+| rewrite | ACCESS EXCLUSIVE rewrite or scan | the table is rewritten or read in full, and its reads and writes block until done |
+| expand | EXPAND AND CONTRACT | no in-place change keeps old readers working; the plan refuses it |
+| drop | drop | the object and, for a table, its data are gone |
+
+Metadata only does not mean free. Most `ALTER TABLE` forms take `ACCESS EXCLUSIVE`, which queues behind every running query and then blocks every query behind it, so a short change can stall a busy table if a long transaction is open. The class says no rows move; the lock still has to be won.
+
+## What the rules say
+
+Every rule has an id in the SQLPG2xx range, names its class and cites the Postgres 18 documentation page behind it, with the differences across majors 14 to 18 where there are any. The full table is on the "Planning and the Change Classifier" page of the sql docs. The common cases:
+
+| Change | Rule | Class | Notes |
+|---|---|---|---|
+| add a column, no default or a non-volatile default | SQLPG201 | metadata | the default is stored once, in every supported major |
+| add a column with a volatile default, a STORED generated column or an identity column | SQLPG202 | rewrite | `gen_random_uuid()`, `random()`, `nextval()`, `clock_timestamp()` are volatile; `now()` is not |
+| add a NOT NULL column with no default | SQLPG203 | expand | fails on a table with rows: add it nullable, backfill, then set NOT NULL |
+| drop a column | SQLPG204 | metadata | the data is gone |
+| rename a column | SQLPG205 | expand | old readers fail the moment it runs |
+| column type to a binary-coercible type (longer varchar, text, wider numeric) | SQLPG206 | metadata | indexes on the column may still be rebuilt |
+| column type with a rewrite (integer to bigint, a shorter varchar) | SQLPG207 | rewrite | |
+| column type across kinds (text to integer) | SQLPG208 | expand | needs USING and breaks readers |
+| set NOT NULL | SQLPG210 | rewrite | a valid `CHECK (col IS NOT NULL)` already in place makes it metadata |
+| add a CHECK | SQLPG218 | rewrite | declare it `NOT VALID`, then validate |
+| add a foreign key | SQLPG219 | validate | `SHARE ROW EXCLUSIVE` on both tables for the scan; `NOT VALID` first avoids it |
+| add a constraint `NOT VALID` | SQLPG217 | metadata | checks new rows only |
+| validate a constraint | SQLPG220 | validate | `SHARE UPDATE EXCLUSIVE`, so reads and writes go on |
+| add a primary key or unique constraint | SQLPG221 | concurrently | build the unique index CONCURRENTLY, then `ADD CONSTRAINT ... USING INDEX` |
+| create an index CONCURRENTLY | SQLPG240 | concurrently | cannot run inside a transaction block; a failed build leaves an INVALID index to drop |
+| create an index without CONCURRENTLY on an existing table | SQLPG241 | concurrently | `SHARE` blocks every write until the build ends: declare it CONCURRENTLY |
+| drop or change an index | SQLPG242, SQLPG243 | concurrently | a changed index is dropped and created |
+| change a view's query, keeping its columns | SQLPG250 | metadata | |
+| change a view's columns | SQLPG251 | expand | dropped and created, with every view on top of it |
+| add an enum label | SQLPG260 | metadata | the new label cannot be used in the same transaction |
+| remove or reorder enum labels | SQLPG261 | expand | a new type, columns moved, the old type dropped |
+| change partitioning or inheritance | SQLPG227 | expand | a new table is filled and swapped in |
+| rename an object or move it to another schema | SQLPG228 | expand | |
+| drop an object | SQLPG270 | drop | |
+
+A table created in the same plan as its indexes needs no CONCURRENTLY, so those indexes classify as create.
+
+Two forms to prefer over the one-step version, because the rule names them:
+
+- Adding a constraint that scans the table: declare it `NOT VALID` first (SQLPG217, metadata), then drop `NOT VALID` to validate it (SQLPG220, validate).
+- Adding a unique key to a busy table: a named unique index `CONCURRENTLY` first, then the constraint over it.
+
+## Identity and renames
+
+Between two builds an object is its export name, so a changed SQL name under the same export is a rename. Against a server, which has no export names, an object is its qualified name (`app.orders`). A rename is expand and contract (SQLPG205, SQLPG228), so plan it deliberately. To tell the plan that a name changed rather than that one object was dropped and another created, put a comment before the CREATE:
+
+```ts
+export const orders = table`
+  -- previously: purchases
+  CREATE TABLE ${app}.orders (...)`;
+```
+
+A column is its name. Declare a column rename on its own line:
+
+```ts
+  status text NOT NULL, -- previously: state
+```
+
+Without the hint, a rename plans as a drop and an add, and the report says so when the two have the same type in the same place. A drop is destructive, so add the hint. An index renames in place (SQLPG229, metadata) and nothing reads an index by name.
+
+## Objects that are not yours
+
+Some objects on a server belong to another tool. The catalog reader marks them foreign, and the plan never proposes to drop them; it reports a hint that the object is kept by that tool and is left alone.
+
+- ORM and migration-runner tables, by name: `_prisma_migrations` (Prisma Migrate), `django_migrations`, `alembic_version`, `__drizzle_migrations`, `flyway_schema_history`, `goose_db_version`, `knex_migrations`, `SequelizeMeta`, `pgmigrations`, `schema_migrations`, `ar_internal_metadata`.
+- Provider-owned objects: with `sql.provider` (or a profile's `provider`) set, the provider's own schemas and extensions read as foreign, so a Supabase `auth` schema is not an orphan to drop.
+
+A foreign object is not a change to approve. If a plan wants to drop something you did not declare and it is not marked foreign, it is yours to either declare or leave out of `schemas`.
+
+## No false drift
+
+The server prints DDL its own way (`format_type()`, schema-qualified names, normalized expressions, a constraint name it generated). The plan undoes those rewrites with normalization rules and asks the server about any expression the rules leave different, so formatting alone is not a change. A constraint the declaration leaves unnamed is matched by what it says, so the name Postgres gave it is not a change either. Name a constraint when a later migration must refer to it.
+
+`chant lifecycle plan` reports each change's disruption: create and metadata are `in-place`; validate, concurrently and rewrite are `rolling`; expand and contract is `replace`; a drop is `destroy`. For a column, constraint, view or enum change the lifecycle plan reports `unknown`, because the class depends on both definitions; `chant sql plan` classifies those.
+
+## Reading a plan in a pull request
+
+1. `chant build src --lexicon sql -o head.json` on the pull request branch, and the same on the base branch into `base.json`.
+2. `chant sql diff base.json head.json`.
+3. Read the `Warning` line: it counts the rewrites, each of which blocks the table's reads and writes until done. The rule names the form that does not.
+4. An exit of 2 means a refused change. Do not merge it as an ordinary change: it runs as expand and contract (add the new, write both, backfill, move readers, drop the old). For a column rename (SQLPG205) or a type change across kinds (SQLPG208) the report names the `PostgresMigrationOp` declaration to add (`migrationOps` in `--json`); run it with `chant run <name>` until it is done (see `chant-sql-postgres-migration`). The other expand-and-contract changes have no Op yet.
+
+## Applying
+
+`ApplyOp` with `target: "postgres"` applies a build (`dist/schema.json`) to the server `sql.profiles.<env>` binds, through `postgresApply`.
+
+```ts
+import { ApplyOp } from "@intentius/chant/op";
+
+const { op } = ApplyOp({ name: "schema-apply", env: "prod", target: "postgres", delete: "gated" });
+```
+
+The applier sends the statements for every change the classifier does not refuse, in the build's order, and a refused change is never sent.
+
+Transactions follow the class, so a long scan holds locks on one table and what came before has committed:
+
+- consecutive catalog-only statements (create, metadata, drop) share one transaction, across objects;
+- a statement that reads or rewrites rows (rewrite and validate classes, an index built without `CONCURRENTLY`) runs in a transaction of its own object;
+- `CREATE INDEX CONCURRENTLY`, `DROP INDEX CONCURRENTLY` and `ALTER TYPE ... ADD VALUE` run alone, outside any transaction. A failed `CONCURRENTLY` build leaves an INVALID index; the applier drops it, and only an invalid one.
+
+A primary key or unique constraint added to an existing table goes in two steps, the unique index built `CONCURRENTLY` and then `ADD CONSTRAINT ... USING INDEX` (SQLPG221), so a busy table is never blocked for the build.
+
+Timeouts are set on every statement: `lock_timeout` 5000 ms, so a statement that cannot get its lock fails instead of queueing behind a long transaction and blocking everything behind it; `statement_timeout` 60000 ms for catalog-only statements, and 0 (no limit) for scans, rewrites and `CONCURRENTLY` builds. Set them per environment with `lockTimeoutMs`, `statementTimeoutMs` and `scanTimeoutMs` on `sql.profiles.<env>`, or per call.
+
+```ts
+profiles: { prod: { url: "postgres://db.internal:5432/shop", lockTimeoutMs: 2000, scanTimeoutMs: 600000 } }
+```
+
+Each object gets a verdict:
+
+- created, updated or unchanged;
+- not attempted `unsupported-kind` for an expand-and-contract change. The detail names each rule, its restriction and the Postgres 18 page, and for a column rename or a type change across kinds the `PostgresMigrationOp` declaration. Also a change with no in-place statement on the target major;
+- not attempted `filtered` for a column drop when the Op may not delete, and for an object another tool keeps (an ORM revision table, a provider schema);
+- not attempted `dependency-failed` when a referenced object is not on the server, or when the object's statements ran in a transaction that rolled back;
+- not attempted `no-binding` or `no-credentials` when the environment names no server or the credentials are missing.
+
+When the server refuses a statement, that object goes to `failed`, the objects that depend on it are `dependency-failed`, and the apply throws `PostgresApplyError` carrying the outcome. The outcome records the timeouts, every statement with the values it ran under and its transaction, and how each transaction ended. A lock timeout reads `lock_timeout 5000ms: another session holds a lock this statement needs; SQLSTATE 55P03`: run again when the blocker has finished. A rolled-back transaction changed nothing, so a re-run starts clean.
+
+The target major is the build output's `postgresMajor`, else `sql.postgresMajor`, and picks `SET EXPRESSION` (17 and later) and `SET STORAGE DEFAULT` (16 and later).
+
+Ownership is a trailer on the object's comment, set with `COMMENT ON` in the transaction that creates it: `One row per account [chant managed-by=chant stack=shop env=prod]`. It replaces the declared object comment on create; column and constraint comments are sent as written. Prune drops owned objects in the declared schemas only: views, indexes (`CONCURRENTLY`), tables, sequences, types, extensions, and schemas last. It never uses `CASCADE`, and an object something else depends on is reported `not-prunable`. Objects without the trailer, and other stacks', are never touched.
+
+To try an apply without a real server, `chant emulator up --lexicon sql` starts the pinned `postgres` image and reports `POSTGRES_URL`, `POSTGRES_USER` and `POSTGRES_PASSWORD`, which the binding reads when no profile names a server.
+
+## Not covered
+
+Roles, grants, policies, functions and triggers are not modelled, so a plan does not see them. `chant migrate` is not a schema migration runner.