@intentius/chant-lexicon-sql 0.114.1 → 0.116.0

package/src/postgres/live/routines.e2e.test.ts

@@ -0,0 +1,173 @@
+/**
+ * Functions, procedures and triggers against a live server (#3680): applied
+ * as declared, they plan with no change and read back with no drift; a body,
+ * a result type and a trigger's condition changed in the declaration are
+ * planned with their rules and applied; a function replaced by hand is
+ * drift; and an import writes them back as `func`, `procedure` and
+ * `trigger` declarations that parse.
+ *
+ * The server: `CHANT_SQL_E2E_POSTGRES_URL` (a `postgres://user@host:port/db`
+ * URL, with `CHANT_SQL_E2E_POSTGRES_PASSWORD`) names a running one, such as
+ * the one `chant emulator up --lexicon sql` starts. Without it, a throwaway
+ * server at the pin, skipped cleanly when Docker is not available. The test
+ * writes only to the schema `chant_e2e_3680`, dropped afterwards.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { dockerAvailable, startTestPostgres, type TestPostgres } from "../testing/server";
+import { connectPostgres, type PostgresClient, type PostgresEndpoint } from "./client";
+import { readLiveSchema } from "./catalog";
+import { planPgAgainstServer } from "../plan/commands";
+import { observeResourcesDeep } from "../plan/deep";
+import { postgresApply } from "../../op/activities/postgres-apply";
+import { objectsToIR } from "../import/ir";
+import { PostgresGenerator } from "../import/generator";
+import { liveProps } from "../plan/deep";
+import { declaredObjects } from "../apply/apply";
+
+const SCHEMA = "chant_e2e_3680";
+const given = process.env.CHANT_SQL_E2E_POSTGRES_URL;
+const enabled = given ? true : await dockerAvailable();
+const dir = mkdtempSync(join(tmpdir(), "chant-pg-routines-"));
+let server: TestPostgres | undefined;
+let endpoint: PostgresEndpoint;
+let admin: PostgresClient;
+
+beforeAll(async () => {
+  if (!enabled) return;
+  if (given) endpoint = { url: given, password: process.env.CHANT_SQL_E2E_POSTGRES_PASSWORD ?? "" };
+  else {
+    server = await startTestPostgres();
+    endpoint = server.endpoint();
+  }
+  admin = await connectPostgres(endpoint);
+  await admin.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`);
+}, 600_000);
+
+afterAll(async () => {
+  if (enabled) {
+    await admin?.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`).catch(() => undefined);
+    await admin?.end();
+  }
+  await server?.stop();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+interface Obj {
+  export: string;
+  type: string;
+  ddl: string;
+  dependsOn?: string[];
+}
+
+const MARKER = { stack: "e2e3680", env: "test" };
+const profile = () => ({
+  config: { ownership: MARKER, sql: { profiles: { e2e: { url: endpoint.url, password: { env: "PG_E2E_PASSWORD" }, schemas: [SCHEMA] } } } },
+  env: { PG_E2E_PASSWORD: endpoint.password ?? "" },
+});
+
+function writeBuild(file: string, objects: Obj[]): string {
+  const path = join(dir, file);
+  writeFileSync(path, JSON.stringify({ dialect: "postgres", postgresMajor: 18, objects: objects.map((o) => ({ dependsOn: [], ...o })) }));
+  return path;
+}
+
+const apply = (path: string) => postgresApply({ buildPath: path, environment: "e2e" }, undefined, { ...profile(), log: () => undefined });
+const plan = (path: string) => planPgAgainstServer("e2e", path, profile());
+
+const S = SCHEMA;
+const objects = (v: { body: string; totalReturns: string; when: string }): Obj[] => [
+  { export: "app", type: "Postgres::Schema", ddl: `CREATE SCHEMA ${S}` },
+  {
+    export: "orders",
+    type: "Postgres::Table",
+    dependsOn: ["app"],
+    ddl: `CREATE TABLE ${S}.orders (id bigint PRIMARY KEY, status text NOT NULL, amount numeric(12,2), note varchar(40), updated_at timestamptz)`,
+  },
+  {
+    export: "touch",
+    type: "Postgres::Function",
+    dependsOn: ["app"],
+    ddl: `CREATE FUNCTION ${S}.touch() RETURNS trigger LANGUAGE plpgsql SET search_path = ${S} AS $$\nBEGIN\n  ${v.body}\n  RETURN NEW;\nEND\n$$;\nCOMMENT ON FUNCTION ${S}.touch() IS 'Stamps updated_at'`,
+  },
+  {
+    export: "total",
+    type: "Postgres::Function",
+    dependsOn: ["orders"],
+    ddl: `CREATE FUNCTION ${S}.total(min_amount numeric DEFAULT 0, label varchar(10) = 'all') RETURNS ${v.totalReturns} LANGUAGE sql STABLE PARALLEL SAFE AS $$ SELECT coalesce(sum(amount), 0)::${v.totalReturns} FROM ${S}.orders WHERE amount >= min_amount $$`,
+  },
+  {
+    export: "archive",
+    type: "Postgres::Procedure",
+    dependsOn: ["orders"],
+    ddl: `CREATE PROCEDURE ${S}.archive(IN before_id bigint, INOUT archived int DEFAULT NULL) LANGUAGE plpgsql AS $$ BEGIN DELETE FROM ${S}.orders WHERE id < before_id; GET DIAGNOSTICS archived = ROW_COUNT; END $$`,
+  },
+  {
+    export: "ordersTouch",
+    type: "Postgres::Trigger",
+    dependsOn: ["orders", "touch"],
+    ddl: `CREATE TRIGGER orders_touch BEFORE INSERT OR UPDATE OF status, note ON ${S}.orders FOR EACH ROW WHEN (${v.when}) EXECUTE FUNCTION ${S}.touch('x', 1)`,
+  },
+];
+
+const V1 = { body: "NEW.updated_at := now();", totalReturns: "numeric", when: "NEW.note IS DISTINCT FROM 'skip'" };
+const V2 = { body: "NEW.updated_at := clock_timestamp();", totalReturns: "double precision", when: "NEW.note <> 'skip' AND NEW.status = 'open'" };
+
+describe.skipIf(!enabled)("functions, procedures and triggers on a live server", () => {
+  test("apply, plan, drift, change and import", async () => {
+    const v1 = writeBuild("v1.json", objects(V1));
+    const first = await apply(v1);
+    expect(first.failed).toEqual([]);
+    expect(first.applied.map((a) => [a.name, a.action])).toEqual([
+      [S, "created"],
+      [`${S}.orders`, "created"],
+      [`${S}.touch()`, "created"],
+      [`${S}.total(numeric,character varying)`, "created"],
+      [`${S}.archive(bigint,integer)`, "created"],
+      [`orders_touch ON ${S}.orders`, "created"],
+    ]);
+    // The trigger fires, with its WHEN and its arguments.
+    await admin.query(`INSERT INTO ${S}.orders (id, status, amount) VALUES (1, 'open', 10), (2, 'open', 5)`);
+    expect((await admin.query<{ n: string }>(`SELECT count(*) AS n FROM ${S}.orders WHERE updated_at IS NOT NULL`))[0]!.n).toBe("2");
+
+    // As declared: no change, and no drift, the WHEN the server prints with its casts included.
+    expect((await plan(v1)).changes).toEqual([]);
+    const declared = declaredObjects(JSON.stringify({ dialect: "postgres", objects: objects(V1).map((o) => ({ dependsOn: [], ...o })) }));
+    const entities = new Map(declared.map((o) => [o.exportName, { entityType: o.type, props: o.props }]));
+    const deep = await observeResourcesDeep({ environment: "e2e", entityNames: [...entities.keys()], entities, ...profile() });
+    expect(deep.unobserved ?? {}).toEqual({});
+    const strip = (p: Record<string, unknown>) => Object.fromEntries(Object.entries(p).filter(([k]) => !["ddl", "source", "reads", "orReplace"].includes(k)));
+    for (const [name, e] of entities) expect(strip(deep.resources[name]!.properties), name).toEqual(strip(e.props));
+
+    // Replaced by hand: drift, in the plan and in the deep read.
+    await admin.query(`CREATE OR REPLACE FUNCTION ${S}.touch() RETURNS trigger LANGUAGE plpgsql SET search_path = ${S} AS $$ BEGIN RETURN NEW; END $$`);
+    expect((await plan(v1)).changes.map((c) => [c.object, c.field, c.rule])).toEqual([[`touch (${S}.touch())`, "body", "SQLPG280"]]);
+    const drifted = await observeResourcesDeep({ environment: "e2e", entityNames: ["touch"], entities, ...profile() });
+    expect(drifted.resources.touch!.properties.body).toBe(" BEGIN RETURN NEW; END ");
+
+    // Changed in the declaration: a body (OR REPLACE), a result type (drop and create), a trigger's condition (OR REPLACE).
+    const v2 = writeBuild("v2.json", objects(V2));
+    expect((await plan(v2)).changes.map((c) => [c.object, c.rule])).toEqual([
+      [`touch (${S}.touch())`, "SQLPG280"],
+      [`total (${S}.total(numeric,character varying))`, "SQLPG281"],
+      [`ordersTouch (${S}.orders_touch ON ${S}.orders)`, "SQLPG284"],
+    ]);
+    const second = await apply(v2);
+    expect(second.failed).toEqual([]);
+    expect(second.applied.filter((a) => a.action === "updated").map((a) => a.name)).toEqual([`${S}.touch()`, `${S}.total(numeric,character varying)`, `orders_touch ON ${S}.orders`]);
+    expect((await plan(v2)).changes).toEqual([]);
+    expect((await admin.query<{ c: string }>(`SELECT obj_description('${S}.total(numeric,varchar)'::regprocedure, 'pg_proc') AS c`))[0]!.c).toContain("stack=e2e3680");
+
+    // Import: the server's own printing, written back as declarations that parse.
+    const live = await readLiveSchema(admin, { schemas: [S] });
+    for (const o of live) liveProps(o);
+    const [file] = new PostgresGenerator().generate(objectsToIR(live.map((o) => ({ type: o.type, schema: o.schema, name: o.name, ddl: o.statement }))));
+    expect(file!.content).toContain("export const touch = func`");
+    expect(file!.content).toContain("export const archive = procedure`");
+    expect(file!.content).toContain("export const ordersTouch = trigger`");
+    expect(file!.content).toContain("EXECUTE FUNCTION ${touch}('x', '1')");
+  }, 600_000);
+});

package/src/postgres/migrate/batches.e2e.test.ts

@@ -96,7 +96,8 @@ describe.skipIf(!enabled)("a uuid primary key", () => {
     await admin!.query("CREATE FUNCTION audit.count_update() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN INSERT INTO audit.updates VALUES (NEW.id); RETURN NEW; END $$");
     await admin!.query("CREATE TRIGGER audit_update AFTER UPDATE ON uu.tokens FOR EACH ROW EXECUTE FUNCTION audit.count_update()");
     writeBuild("tokens-v2.json", [UU, tokens("integer")]);
-    expect((await plan("tokens-v2.json")).changes.map((c) => c.rule)).toEqual(["SQLPG208"]);
+    // The application's own trigger is not declared, so the plan also proposes its drop (SQLPG285), which only a prune makes.
+    expect((await plan("tokens-v2.json")).changes.map((c) => c.rule)).toEqual(["SQLPG208", "SQLPG285"]);
 
     expect(await m.postgresMigrationPlan(args(), undefined, deps())).toMatchObject({ state: "migrate", summary: expect.stringContaining("batched by id") });
     await m.postgresMigrationExpand(args(), undefined, deps());

package/src/postgres/migrate/migrate.e2e.test.ts

@@ -307,7 +307,8 @@ describe.skipIf(!enabled)("an interrupted backfill resumes from its receipts", (
     writeBuild("events-v2.json", [EV, V2]);
 
     // varchar(20) to varchar(10) is a rewrite in place (SQLPG207), which the Op makes in batches instead.
-    expect((await plan("events-v2.json")).changes.map((c) => c.rule)).toEqual(["SQLPG207"]);
+    // The application's own trigger is not declared, so the plan also proposes its drop (SQLPG285), which only a prune makes.
+    expect((await plan("events-v2.json")).changes.map((c) => c.rule)).toEqual(["SQLPG207", "SQLPG285"]);
     expect(await m.postgresMigrationPlan(args(), undefined, deps())).toMatchObject({ state: "migrate", change: "type" });
     await m.postgresMigrationExpand(args(), undefined, deps());
     await m.postgresMigrationDualWrite(args(), undefined, deps());

package/src/postgres/migrate/migrate.test.ts

@@ -102,12 +102,13 @@ describe("PostgresMigrationOp", () => {
     }
   });
 
-  test("every migration step is an activity the sql lexicon exports, and none is a receipt activity", () => {
+  test("every migration step is an activity the sql lexicon exports, and the receipt activities are only fallbacks", () => {
     const exported = new Set(Object.entries(activitiesModule).filter(([, v]) => typeof v === "function").map(([k]) => k));
     const steps = propsOf(BASE).phases.flatMap((p) => p.steps).filter((s) => s.kind === "activity" && (s as { fn: string }).fn.startsWith("postgresMigration"));
     expect(steps).toHaveLength(9);
     for (const s of steps) expect(exported.has((s as { fn: string }).fn)).toBe(true);
-    expect(exported.has("receiptRead")).toBe(false);
+    // The receipt activities are exported for effect() steps only as fallbacks (#3657), so they never take over another lexicon's receipt row.
+    expect((activitiesModule as { ACTIVITY_FALLBACKS?: readonly string[] }).ACTIVITY_FALLBACKS).toEqual(["receiptRead", "receiptWrite", "receiptStaleness"]);
   });
 });
 

package/src/postgres/migrate/receipts.ts

@@ -27,9 +27,9 @@
  * stale; onFailure and the contract also delete the migration's receipts.
  *
  * The store implements the shared core's `SqlReceiptStore`
- * (`../../core/receipts.ts`) over the migration's own connection. It is not
- * exported as the `receiptRead` / `receiptWrite` activities, whose names are
- * global to a run and belong to an aws or k8s project's receipt row.
+ * (`../../core/receipts.ts`) over the migration's own connection, which the
+ * backfill reaches directly. A sql project's `effect()` receipts use the same
+ * store over `chant_receipts.receipts` (`../../receipts.ts`).
  */
 
 import { OWNERSHIP_MANAGED_BY_VALUE } from "@intentius/chant/ownership";
@@ -48,10 +48,10 @@ export { receiptAddress } from "../../core/receipts";
 
 /** A receipts table chant did not make: a user's table under the same name. */
 export class ReceiptsTableConflict extends Error {
-  constructor(schema: string) {
+  constructor(schema: string, table: string = POSTGRES_RECEIPTS_TABLE) {
     super(
-      `${schema}.${POSTGRES_RECEIPTS_TABLE} exists and is not chant's receipts table (its comment carries no receipts marker). ` +
-        `The migration keeps its receipts under that name, so it stops rather than write into it; rename the table.`,
+      `${schema}.${table} exists and is not chant's receipts table (its comment carries no receipts marker). ` +
+        `chant keeps its receipts under that name, so it stops rather than write into it; rename the table.`,
     );
     this.name = "ReceiptsTableConflict";
   }
@@ -70,10 +70,18 @@ export interface PostgresReceiptStore extends SqlReceiptStore {
  * A receipt store in `<schema>.__chant_receipts`, over `client`. `write` runs
  * on the client as it is, so a write inside the caller's transaction commits
  * or rolls back with it. `identity` is the project's ownership stack and env,
- * which every address starts with.
+ * which every address starts with. `table` names the table in `schema`,
+ * `__chant_receipts` by default (`../../receipts.ts` keeps `effect()`
+ * receipts in `chant_receipts.receipts`).
  */
-export function postgresReceiptStore(client: PostgresClient, schema: string, identity: { stack?: string; env?: string }, opts: { runId?: string } = {}): PostgresReceiptStore {
-  const table = `${quoteIdent(schema)}.${quoteIdent(POSTGRES_RECEIPTS_TABLE)}`;
+export function postgresReceiptStore(
+  client: PostgresClient,
+  schema: string,
+  identity: { stack?: string; env?: string },
+  opts: { runId?: string; table?: string } = {},
+): PostgresReceiptStore {
+  const name = opts.table ?? POSTGRES_RECEIPTS_TABLE;
+  const table = `${quoteIdent(schema)}.${quoteIdent(name)}`;
   let ensured = false;
   const comment = async (): Promise<string | null | undefined> => {
     const [row] = await client.query<{ present: boolean; comment: string | null }>(
@@ -87,7 +95,7 @@ export function postgresReceiptStore(client: PostgresClient, schema: string, ide
     if (ensured) return true;
     const c = await comment();
     if (c === undefined) return false;
-    if (!hasChantTrailerKey(c ?? undefined, [RECEIPTS_TRAILER_KEY])) throw new ReceiptsTableConflict(schema);
+    if (!hasChantTrailerKey(c ?? undefined, [RECEIPTS_TRAILER_KEY])) throw new ReceiptsTableConflict(schema, name);
     return true;
   };
   const address = (receipt: EffectReceiptRef) => receiptAddress(identity, receipt.effect);