@intentius/chant-lexicon-sql 0.115.0 → 0.116.0

package/src/postgres/live/catalog.ts

@@ -18,6 +18,11 @@
  * - objects an extension owns (`pg_depend.deptype = 'e'`): the extension
  *   declares them, so the extension is what is imported;
  * - `plpgsql`, installed in every database;
+ * - aggregates, and a trigger the server makes for itself (a foreign key's)
+ *   or clones onto a partition from its parent's;
+ * - a routine with a SQL-standard body (`RETURN`, `BEGIN ATOMIC`): read, but
+ *   marked `unsupported`, since the server prints it rewritten and a
+ *   declaration cannot hold it;
  * - a sequence an identity column owns (it is the column's), and a `serial`
  *   column's sequence, since the column prints as `serial`;
  * - an index that backs a constraint (the constraint creates it), and an
@@ -76,6 +81,12 @@ export interface LivePgObject {
   statement: string;
   /** The tool whose bookkeeping this is, when it is another tool's (an ORM's revision table). */
   foreign?: string;
+  /** A routine's input parameter types, `(integer,text)`; a trigger's table, ` ON app.users` (`../plan/normalize.ts`). */
+  signature?: string;
+  /** What depends on a routine (a view, a trigger, a column default), as the server describes each. */
+  dependents?: string[];
+  /** Why no declaration can hold this object; it is read so import can say it left it out. */
+  unsupported?: string;
 }
 
 /** What {@link readLiveSchema} reads. */
@@ -181,8 +192,9 @@ const seqParams = (r: Row): SequenceParams => ({
 
 /**
  * Every object in scope, in a stable order: schemas, extensions, enums,
- * domains, sequences, tables, views and materialized views, indexes; each
- * kind by schema then name, in byte order.
+ * domains, sequences, tables, views and materialized views, indexes,
+ * functions and procedures, triggers; each kind by schema then name, in byte
+ * order.
  */
 export async function readLiveSchema(client: PostgresClient, scope: SchemaScope = {}): Promise<LivePgObject[]> {
   const out: LivePgObject[] = [];
@@ -556,11 +568,73 @@ export async function readLiveSchema(client: PostgresClient, scope: SchemaScope
     }
   }
 
+  // Functions and procedures (not aggregates).
+  {
+    const rows = await client.query<Row>(
+      `SELECT p.oid::text AS oid, n.nspname AS schema, p.proname AS name, p.prokind AS kind, pg_catalog.pg_get_functiondef(p.oid) AS def,
+              '(' || pg_catalog.replace(pg_catalog.oidvectortypes(p.proargtypes), ', ', ',') || ')' AS signature,
+              p.prosqlbody IS NOT NULL AS sqlbody, pg_catalog.obj_description(p.oid, 'pg_proc') AS comment,
+              ARRAY(SELECT pg_catalog.pg_describe_object(d.classid, d.objid, d.objsubid) FROM pg_catalog.pg_depend d
+                    WHERE d.refclassid = 'pg_catalog.pg_proc'::pg_catalog.regclass AND d.refobjid = p.oid AND d.deptype = 'n'
+                    ORDER BY 1) AS dependents
+       FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
+       WHERE p.prokind IN ('f', 'p', 'w') AND ${s.sql} AND ${notExtensionOwned("pg_proc", "p.oid")}
+       ORDER BY n.nspname ${C}, p.proname ${C}, pg_catalog.oidvectortypes(p.proargtypes) ${C}`,
+      s.params,
+    );
+    for (const r of rows) {
+      // A migration's dual-write function, until its contract.
+      if (hasChantTrailerKey(str(r.comment), [MIGRATION_TRAILER_KEY])) continue;
+      const signature = String(r.signature);
+      const q = `${qname(String(r.schema), String(r.name))}${signature}`;
+      const procedure = r.kind === "p";
+      const dependents = (r.dependents as string[] | null) ?? [];
+      out.push({
+        type: procedure ? POSTGRES_ENTITY_TYPES.procedure : POSTGRES_ENTITY_TYPES.function,
+        schema: String(r.schema),
+        name: String(r.name),
+        oid: String(r.oid),
+        signature,
+        ...(dependents.length > 0 ? { dependents } : {}),
+        ...(str(r.comment) ? { comment: str(r.comment) } : {}),
+        ...(r.sqlbody === true ? { unsupported: "its body is SQL-standard (RETURN or BEGIN ATOMIC), which the server prints back rewritten; declare it with a string body (AS $$ ... $$)" } : {}),
+        statement: [String(r.def).trim(), ...commentOn(`${procedure ? "PROCEDURE" : "FUNCTION"} ${q}`, r.comment)].join(";\n"),
+      });
+    }
+  }
+
+  // Triggers: the ones written by hand, not a foreign key's or a partition's clone of its parent's.
+  {
+    const rows = await client.query<Row>(
+      `SELECT t.oid::text AS oid, n.nspname AS schema, c.relname AS table_name, t.tgname AS name, pg_catalog.pg_get_triggerdef(t.oid, true) AS def,
+              pg_catalog.obj_description(t.oid, 'pg_trigger') AS comment
+       FROM pg_catalog.pg_trigger t JOIN pg_catalog.pg_class c ON c.oid = t.tgrelid JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+       WHERE NOT t.tgisinternal AND t.tgparentid = 0 AND ${s.sql} AND ${notExtensionOwned("pg_class", "c.oid")}
+       ORDER BY n.nspname ${C}, c.relname ${C}, t.tgname ${C}`,
+      s.params,
+    );
+    for (const r of rows) {
+      // A migration's dual-write trigger, until its contract.
+      if (hasChantTrailerKey(str(r.comment), [MIGRATION_TRAILER_KEY])) continue;
+      const table = qname(String(r.schema), String(r.table_name));
+      out.push({
+        type: POSTGRES_ENTITY_TYPES.trigger,
+        schema: String(r.schema),
+        name: String(r.name),
+        oid: String(r.oid),
+        signature: ` ON ${table}`,
+        ...(str(r.comment) ? { comment: str(r.comment) } : {}),
+        ...(FOREIGN_TABLES[String(r.table_name)] ? { foreign: FOREIGN_TABLES[String(r.table_name)] } : {}),
+        statement: [String(r.def), ...commentOn(`TRIGGER ${quoteIdent(String(r.name))} ON ${table}`, r.comment)].join(";\n"),
+      });
+    }
+  }
+
   return out;
 }
 
-/** The key an object is found by: its type and schema-qualified name. */
-export const liveKey = (type: string, schema: string | undefined, name: string): string => `${type} ${schema ?? ""}.${name}`;
+/** The key an object is found by: its type and schema-qualified name, and a routine's parameter types or a trigger's table. */
+export const liveKey = (type: string, schema: string | undefined, name: string, signature?: string): string => `${type} ${schema ?? ""}.${name}${signature ?? ""}`;
 
 /**
  * Mark what the managed service owns (#3282) as foreign: its reserved

package/src/postgres/live/describe-resources.ts

@@ -27,17 +27,27 @@ import { liveKey, markProviderOwned, readLiveSchema, type LivePgObject } from ".
 import { POSTGRES_ENTITY_TYPES } from "../entity-types";
 import { isChantManaged, readMarker, stripMarker } from "../../core/ownership";
 import { identValue } from "../parser";
+import { canonicalPgObject } from "../plan/normalize";
 
 interface Bound {
   target: PostgresTarget;
   byKey: Map<string, LivePgObject>;
 }
 
-/** The schema and name a declared entity is created as; a bare name is in the default schema. */
-export function declaredAddress(entity: { type: string; props: Record<string, unknown> }, defaultSchema: string): { schema?: string; name: string } {
+/**
+ * The schema and name a declared entity is created as; a bare name is in the
+ * default schema. A routine's parameter types and a trigger's table tell it
+ * from another of the same name (`signature`).
+ */
+export function declaredAddress(entity: { type: string; props: Record<string, unknown> }, defaultSchema: string): { schema?: string; name: string; signature?: string } {
   const name = String(entity.props.name ?? "");
   if (entity.type === POSTGRES_ENTITY_TYPES.schema || entity.type === POSTGRES_ENTITY_TYPES.extension) return { name };
-  return { schema: typeof entity.props.schema === "string" ? entity.props.schema : identValue(defaultSchema), name };
+  const schema = typeof entity.props.schema === "string" ? entity.props.schema : identValue(defaultSchema);
+  if (entity.type === POSTGRES_ENTITY_TYPES.function || entity.type === POSTGRES_ENTITY_TYPES.procedure || entity.type === POSTGRES_ENTITY_TYPES.trigger) {
+    const signature = canonicalPgObject(entity.type, entity.props, defaultSchema).signature;
+    return { schema, name, ...(signature !== undefined ? { signature } : {}) };
+  }
+  return { schema, name };
 }
 
 /** A schema scope for the catalog read: the profile's schemas, else every schema the declarations name and the default one. */
@@ -59,7 +69,7 @@ function adapter(options: BindOptions & { owned?: boolean; declared: DeclaredEnt
       const { target, client } = await bindPostgres(options);
       try {
         const live = markProviderOwned(await readLiveSchema(client, { schemas: scopeFor(target, options.declared.filter((d) => d.type.startsWith("Postgres::"))) }), target.provider);
-        return { target, byKey: new Map(live.map((o) => [liveKey(o.type, o.schema, o.name), o])) };
+        return { target, byKey: new Map(live.map((o) => [liveKey(o.type, o.schema, o.name, o.signature), o])) };
       } finally {
         await client.end();
       }
@@ -67,9 +77,9 @@ function adapter(options: BindOptions & { owned?: boolean; declared: DeclaredEnt
     classifyBindFailure: (err) => classifyPostgresFailure(err),
     async read({ target, byKey }, entity): Promise<EntityObservation> {
       if (!entity.type.startsWith("Postgres::")) return { unobserved: { reason: "unsupported-kind", detail: entity.type } };
-      const { schema, name } = declaredAddress(entity, target.defaultSchema);
-      const queried = `${redactUrl(target.endpoint.url)} ${schema ? `${schema}.` : ""}${name}`;
-      const live = byKey.get(liveKey(entity.type, schema, name));
+      const { schema, name, signature } = declaredAddress(entity, target.defaultSchema);
+      const queried = `${redactUrl(target.endpoint.url)} ${schema ? `${schema}.` : ""}${name}${signature ?? ""}`;
+      const live = byKey.get(liveKey(entity.type, schema, name, signature));
       if (!live) return { absent: true, queried };
       const owned = !live.foreign && isChantManaged(live.comment);
       if (options.owned && !owned) {

package/src/postgres/live/routines.e2e.test.ts

@@ -0,0 +1,173 @@
+/**
+ * Functions, procedures and triggers against a live server (#3680): applied
+ * as declared, they plan with no change and read back with no drift; a body,
+ * a result type and a trigger's condition changed in the declaration are
+ * planned with their rules and applied; a function replaced by hand is
+ * drift; and an import writes them back as `func`, `procedure` and
+ * `trigger` declarations that parse.
+ *
+ * The server: `CHANT_SQL_E2E_POSTGRES_URL` (a `postgres://user@host:port/db`
+ * URL, with `CHANT_SQL_E2E_POSTGRES_PASSWORD`) names a running one, such as
+ * the one `chant emulator up --lexicon sql` starts. Without it, a throwaway
+ * server at the pin, skipped cleanly when Docker is not available. The test
+ * writes only to the schema `chant_e2e_3680`, dropped afterwards.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { dockerAvailable, startTestPostgres, type TestPostgres } from "../testing/server";
+import { connectPostgres, type PostgresClient, type PostgresEndpoint } from "./client";
+import { readLiveSchema } from "./catalog";
+import { planPgAgainstServer } from "../plan/commands";
+import { observeResourcesDeep } from "../plan/deep";
+import { postgresApply } from "../../op/activities/postgres-apply";
+import { objectsToIR } from "../import/ir";
+import { PostgresGenerator } from "../import/generator";
+import { liveProps } from "../plan/deep";
+import { declaredObjects } from "../apply/apply";
+
+const SCHEMA = "chant_e2e_3680";
+const given = process.env.CHANT_SQL_E2E_POSTGRES_URL;
+const enabled = given ? true : await dockerAvailable();
+const dir = mkdtempSync(join(tmpdir(), "chant-pg-routines-"));
+let server: TestPostgres | undefined;
+let endpoint: PostgresEndpoint;
+let admin: PostgresClient;
+
+beforeAll(async () => {
+  if (!enabled) return;
+  if (given) endpoint = { url: given, password: process.env.CHANT_SQL_E2E_POSTGRES_PASSWORD ?? "" };
+  else {
+    server = await startTestPostgres();
+    endpoint = server.endpoint();
+  }
+  admin = await connectPostgres(endpoint);
+  await admin.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`);
+}, 600_000);
+
+afterAll(async () => {
+  if (enabled) {
+    await admin?.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`).catch(() => undefined);
+    await admin?.end();
+  }
+  await server?.stop();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+interface Obj {
+  export: string;
+  type: string;
+  ddl: string;
+  dependsOn?: string[];
+}
+
+const MARKER = { stack: "e2e3680", env: "test" };
+const profile = () => ({
+  config: { ownership: MARKER, sql: { profiles: { e2e: { url: endpoint.url, password: { env: "PG_E2E_PASSWORD" }, schemas: [SCHEMA] } } } },
+  env: { PG_E2E_PASSWORD: endpoint.password ?? "" },
+});
+
+function writeBuild(file: string, objects: Obj[]): string {
+  const path = join(dir, file);
+  writeFileSync(path, JSON.stringify({ dialect: "postgres", postgresMajor: 18, objects: objects.map((o) => ({ dependsOn: [], ...o })) }));
+  return path;
+}
+
+const apply = (path: string) => postgresApply({ buildPath: path, environment: "e2e" }, undefined, { ...profile(), log: () => undefined });
+const plan = (path: string) => planPgAgainstServer("e2e", path, profile());
+
+const S = SCHEMA;
+const objects = (v: { body: string; totalReturns: string; when: string }): Obj[] => [
+  { export: "app", type: "Postgres::Schema", ddl: `CREATE SCHEMA ${S}` },
+  {
+    export: "orders",
+    type: "Postgres::Table",
+    dependsOn: ["app"],
+    ddl: `CREATE TABLE ${S}.orders (id bigint PRIMARY KEY, status text NOT NULL, amount numeric(12,2), note varchar(40), updated_at timestamptz)`,
+  },
+  {
+    export: "touch",
+    type: "Postgres::Function",
+    dependsOn: ["app"],
+    ddl: `CREATE FUNCTION ${S}.touch() RETURNS trigger LANGUAGE plpgsql SET search_path = ${S} AS $$\nBEGIN\n  ${v.body}\n  RETURN NEW;\nEND\n$$;\nCOMMENT ON FUNCTION ${S}.touch() IS 'Stamps updated_at'`,
+  },
+  {
+    export: "total",
+    type: "Postgres::Function",
+    dependsOn: ["orders"],
+    ddl: `CREATE FUNCTION ${S}.total(min_amount numeric DEFAULT 0, label varchar(10) = 'all') RETURNS ${v.totalReturns} LANGUAGE sql STABLE PARALLEL SAFE AS $$ SELECT coalesce(sum(amount), 0)::${v.totalReturns} FROM ${S}.orders WHERE amount >= min_amount $$`,
+  },
+  {
+    export: "archive",
+    type: "Postgres::Procedure",
+    dependsOn: ["orders"],
+    ddl: `CREATE PROCEDURE ${S}.archive(IN before_id bigint, INOUT archived int DEFAULT NULL) LANGUAGE plpgsql AS $$ BEGIN DELETE FROM ${S}.orders WHERE id < before_id; GET DIAGNOSTICS archived = ROW_COUNT; END $$`,
+  },
+  {
+    export: "ordersTouch",
+    type: "Postgres::Trigger",
+    dependsOn: ["orders", "touch"],
+    ddl: `CREATE TRIGGER orders_touch BEFORE INSERT OR UPDATE OF status, note ON ${S}.orders FOR EACH ROW WHEN (${v.when}) EXECUTE FUNCTION ${S}.touch('x', 1)`,
+  },
+];
+
+const V1 = { body: "NEW.updated_at := now();", totalReturns: "numeric", when: "NEW.note IS DISTINCT FROM 'skip'" };
+const V2 = { body: "NEW.updated_at := clock_timestamp();", totalReturns: "double precision", when: "NEW.note <> 'skip' AND NEW.status = 'open'" };
+
+describe.skipIf(!enabled)("functions, procedures and triggers on a live server", () => {
+  test("apply, plan, drift, change and import", async () => {
+    const v1 = writeBuild("v1.json", objects(V1));
+    const first = await apply(v1);
+    expect(first.failed).toEqual([]);
+    expect(first.applied.map((a) => [a.name, a.action])).toEqual([
+      [S, "created"],
+      [`${S}.orders`, "created"],
+      [`${S}.touch()`, "created"],
+      [`${S}.total(numeric,character varying)`, "created"],
+      [`${S}.archive(bigint,integer)`, "created"],
+      [`orders_touch ON ${S}.orders`, "created"],
+    ]);
+    // The trigger fires, with its WHEN and its arguments.
+    await admin.query(`INSERT INTO ${S}.orders (id, status, amount) VALUES (1, 'open', 10), (2, 'open', 5)`);
+    expect((await admin.query<{ n: string }>(`SELECT count(*) AS n FROM ${S}.orders WHERE updated_at IS NOT NULL`))[0]!.n).toBe("2");
+
+    // As declared: no change, and no drift, the WHEN the server prints with its casts included.
+    expect((await plan(v1)).changes).toEqual([]);
+    const declared = declaredObjects(JSON.stringify({ dialect: "postgres", objects: objects(V1).map((o) => ({ dependsOn: [], ...o })) }));
+    const entities = new Map(declared.map((o) => [o.exportName, { entityType: o.type, props: o.props }]));
+    const deep = await observeResourcesDeep({ environment: "e2e", entityNames: [...entities.keys()], entities, ...profile() });
+    expect(deep.unobserved ?? {}).toEqual({});
+    const strip = (p: Record<string, unknown>) => Object.fromEntries(Object.entries(p).filter(([k]) => !["ddl", "source", "reads", "orReplace"].includes(k)));
+    for (const [name, e] of entities) expect(strip(deep.resources[name]!.properties), name).toEqual(strip(e.props));
+
+    // Replaced by hand: drift, in the plan and in the deep read.
+    await admin.query(`CREATE OR REPLACE FUNCTION ${S}.touch() RETURNS trigger LANGUAGE plpgsql SET search_path = ${S} AS $$ BEGIN RETURN NEW; END $$`);
+    expect((await plan(v1)).changes.map((c) => [c.object, c.field, c.rule])).toEqual([[`touch (${S}.touch())`, "body", "SQLPG280"]]);
+    const drifted = await observeResourcesDeep({ environment: "e2e", entityNames: ["touch"], entities, ...profile() });
+    expect(drifted.resources.touch!.properties.body).toBe(" BEGIN RETURN NEW; END ");
+
+    // Changed in the declaration: a body (OR REPLACE), a result type (drop and create), a trigger's condition (OR REPLACE).
+    const v2 = writeBuild("v2.json", objects(V2));
+    expect((await plan(v2)).changes.map((c) => [c.object, c.rule])).toEqual([
+      [`touch (${S}.touch())`, "SQLPG280"],
+      [`total (${S}.total(numeric,character varying))`, "SQLPG281"],
+      [`ordersTouch (${S}.orders_touch ON ${S}.orders)`, "SQLPG284"],
+    ]);
+    const second = await apply(v2);
+    expect(second.failed).toEqual([]);
+    expect(second.applied.filter((a) => a.action === "updated").map((a) => a.name)).toEqual([`${S}.touch()`, `${S}.total(numeric,character varying)`, `orders_touch ON ${S}.orders`]);
+    expect((await plan(v2)).changes).toEqual([]);
+    expect((await admin.query<{ c: string }>(`SELECT obj_description('${S}.total(numeric,varchar)'::regprocedure, 'pg_proc') AS c`))[0]!.c).toContain("stack=e2e3680");
+
+    // Import: the server's own printing, written back as declarations that parse.
+    const live = await readLiveSchema(admin, { schemas: [S] });
+    for (const o of live) liveProps(o);
+    const [file] = new PostgresGenerator().generate(objectsToIR(live.map((o) => ({ type: o.type, schema: o.schema, name: o.name, ddl: o.statement }))));
+    expect(file!.content).toContain("export const touch = func`");
+    expect(file!.content).toContain("export const archive = procedure`");
+    expect(file!.content).toContain("export const ordersTouch = trigger`");
+    expect(file!.content).toContain("EXECUTE FUNCTION ${touch}('x', '1')");
+  }, 600_000);
+});

package/src/postgres/migrate/batches.e2e.test.ts

@@ -96,7 +96,8 @@ describe.skipIf(!enabled)("a uuid primary key", () => {
     await admin!.query("CREATE FUNCTION audit.count_update() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN INSERT INTO audit.updates VALUES (NEW.id); RETURN NEW; END $$");
     await admin!.query("CREATE TRIGGER audit_update AFTER UPDATE ON uu.tokens FOR EACH ROW EXECUTE FUNCTION audit.count_update()");
     writeBuild("tokens-v2.json", [UU, tokens("integer")]);
-    expect((await plan("tokens-v2.json")).changes.map((c) => c.rule)).toEqual(["SQLPG208"]);
+    // The application's own trigger is not declared, so the plan also proposes its drop (SQLPG285), which only a prune makes.
+    expect((await plan("tokens-v2.json")).changes.map((c) => c.rule)).toEqual(["SQLPG208", "SQLPG285"]);
 
     expect(await m.postgresMigrationPlan(args(), undefined, deps())).toMatchObject({ state: "migrate", summary: expect.stringContaining("batched by id") });
     await m.postgresMigrationExpand(args(), undefined, deps());

package/src/postgres/migrate/migrate.e2e.test.ts

@@ -307,7 +307,8 @@ describe.skipIf(!enabled)("an interrupted backfill resumes from its receipts", (
     writeBuild("events-v2.json", [EV, V2]);
 
     // varchar(20) to varchar(10) is a rewrite in place (SQLPG207), which the Op makes in batches instead.
-    expect((await plan("events-v2.json")).changes.map((c) => c.rule)).toEqual(["SQLPG207"]);
+    // The application's own trigger is not declared, so the plan also proposes its drop (SQLPG285), which only a prune makes.
+    expect((await plan("events-v2.json")).changes.map((c) => c.rule)).toEqual(["SQLPG207", "SQLPG285"]);
     expect(await m.postgresMigrationPlan(args(), undefined, deps())).toMatchObject({ state: "migrate", change: "type" });
     await m.postgresMigrationExpand(args(), undefined, deps());
     await m.postgresMigrationDualWrite(args(), undefined, deps());