@supabase/pg-delta 1.0.0-alpha.3 → 1.0.0-alpha.30

package/src/core/objects/extension/extension.model.test.ts

@@ -0,0 +1,98 @@
+import { describe, expect, test } from "bun:test";
+import type { Pool } from "pg";
+import { Extension, extractExtensions } from "./extension.model.ts";
+
+describe("Extension", () => {
+  test("stableId returns extension:<name>", () => {
+    const ext = new Extension({
+      name: "plpgsql",
+      schema: "pg_catalog",
+      relocatable: false,
+      version: "1.0",
+      owner: "postgres",
+      comment: null,
+      members: [],
+    });
+    expect(ext.stableId).toBe("extension:plpgsql");
+  });
+
+  test("identityFields returns name", () => {
+    const ext = new Extension({
+      name: "vector",
+      schema: "extensions",
+      relocatable: true,
+      version: "0.7.0",
+      owner: "postgres",
+      comment: "vector type",
+      members: ["type:extensions.vector"],
+    });
+    expect(ext.identityFields).toEqual({ name: "vector" });
+  });
+
+  test("dataFields returns schema, relocatable, version, owner, comment", () => {
+    const ext = new Extension({
+      name: "pgcrypto",
+      schema: "public",
+      relocatable: false,
+      version: "1.3",
+      owner: "postgres",
+      comment: "cryptographic functions",
+      members: ["procedure:public.gen_random_uuid()"],
+    });
+    expect(ext.dataFields).toEqual({
+      schema: "public",
+      relocatable: false,
+      version: "1.3",
+      owner: "postgres",
+      comment: "cryptographic functions",
+    });
+  });
+});
+
+describe("extractExtensions", () => {
+  test("returns empty array when pool returns no rows", async () => {
+    const pool = {
+      query: async () => ({ rows: [] }),
+    } as unknown as Pool;
+    const result = await extractExtensions(pool);
+    expect(result).toEqual([]);
+  });
+
+  test("returns Extension instances for valid rows", async () => {
+    const pool = {
+      query: async () => ({
+        rows: [
+          {
+            name: "plpgsql",
+            schema: "pg_catalog",
+            relocatable: false,
+            version: "1.0",
+            owner: "postgres",
+            comment: null,
+            members: [],
+          },
+          {
+            name: "vector",
+            schema: "extensions",
+            relocatable: true,
+            version: "0.7.0",
+            owner: "postgres",
+            comment: null,
+            members: ["type:extensions.vector", "table:extensions.vector"],
+          },
+        ],
+      }),
+    } as unknown as Pool;
+    const result = await extractExtensions(pool);
+    expect(result).toHaveLength(2);
+    expect(result[0]).toBeInstanceOf(Extension);
+    expect(result[0].name).toBe("plpgsql");
+    expect(result[0].stableId).toBe("extension:plpgsql");
+    expect(result[0].members).toEqual([]);
+    expect(result[1].name).toBe("vector");
+    expect(result[1].members).toEqual([
+      "type:extensions.vector",
+      "table:extensions.vector",
+    ]);
+  });
+});

package/src/core/objects/extension/extension.model.ts

@@ -0,0 +1,280 @@
+import { sql } from "@ts-safeql/sql-tag";
+import type { Pool } from "pg";
+import z from "zod";
+import { BasePgModel } from "../base.model.ts";
+
+/**
+ * All properties exposed by CREATE EXTENSION statement are included in diff output.
+ * https://www.postgresql.org/docs/current/sql-createextension.html
+ *
+ * ALTER EXTENSION statement can be generated for changes to the following properties:
+ *  - version (limited to available ones), schema (only if relocatable)
+ * https://www.postgresql.org/docs/current/sql-alterextension.html
+ *
+ * Adding or dropping member objects are not supported. For eg. pgmq allows detaching
+ * user defined queues by removing its entry from pg_depend. If the detached table
+ * lives in an excluded schema like pg_catalog, it will not be diffed.
+ *
+ * The extension's configuration tables are not diffed.
+ *  - extconfig, extcondition
+ * https://www.postgresql.org/docs/current/catalog-pg-extension.html
+ */
+const extensionPropsSchema = z.object({
+  name: z.string(),
+  schema: z.string(),
+  relocatable: z.boolean(),
+  version: z.string(),
+  owner: z.string(),
+  comment: z.string().nullable(),
+  members: z.array(z.string()),
+});
+
+export type ExtensionProps = z.infer<typeof extensionPropsSchema>;
+
+export class Extension extends BasePgModel {
+  public readonly name: ExtensionProps["name"];
+  public readonly schema: ExtensionProps["schema"];
+  public readonly relocatable: ExtensionProps["relocatable"];
+  public readonly version: ExtensionProps["version"];
+  public readonly owner: ExtensionProps["owner"];
+  public readonly comment: ExtensionProps["comment"];
+  public readonly members: ExtensionProps["members"];
+
+  constructor(props: ExtensionProps) {
+    super();
+
+    // Identity fields
+    this.name = props.name;
+
+    // Data fields
+    this.schema = props.schema;
+    this.relocatable = props.relocatable;
+    this.version = props.version;
+    this.owner = props.owner;
+    this.comment = props.comment;
+    this.members = props.members;
+  }
+
+  get stableId(): `extension:${string}` {
+    // Extension names are unique per database; schema is relocatable
+    return `extension:${this.name}`;
+  }
+
+  get identityFields() {
+    return {
+      name: this.name,
+    };
+  }
+
+  get dataFields() {
+    return {
+      schema: this.schema,
+      relocatable: this.relocatable,
+      version: this.version,
+      owner: this.owner,
+      comment: this.comment,
+    };
+  }
+}
+
+// TODO: fetch extension dependencies so we can determine when to use CASCADE on creation
+export async function extractExtensions(pool: Pool): Promise<Extension[]> {
+  const { rows: extensionRows } = await pool.query<ExtensionProps>(sql`
+  with extension_rows as (
+    select
+      e.oid,
+      quote_ident(e.extname) as name,
+      e.extnamespace::regnamespace::text as schema,
+      e.extrelocatable as relocatable,
+      e.extversion as version,
+      e.extowner::regrole::text as owner,
+      obj_description(e.oid, 'pg_extension') as comment
+    from
+      pg_catalog.pg_extension e
+  ), extension_members_raw as (
+    select
+      er.oid as extension_oid,
+      d.classid,
+      d.objid,
+      d.objsubid
+    from
+      extension_rows er
+      join pg_depend d on d.refclassid = 'pg_extension'::regclass
+        and d.refobjid = er.oid
+    where
+      d.deptype = 'e'
+  ), ids as (
+    select distinct
+      classid,
+      objid,
+      coalesce(nullif(objsubid, 0), 0)::int2 as objsubid
+    from extension_members_raw
+  ), objects as (
+    select 'pg_namespace'::regclass as classid, n.oid as objid, 0::int2 as objsubid,
+          format('schema:%I', n.nspname) as stable_id
+    from pg_namespace n
+    join ids i on i.classid = 'pg_namespace'::regclass and i.objid = n.oid and i.objsubid = 0
+
+    union all
+
+    select 'pg_class'::regclass, c.oid, 0::int2,
+          case c.relkind
+            when 'r' then format('table:%I.%I', ns.nspname, c.relname)
+            when 'p' then format('table:%I.%I', ns.nspname, c.relname)
+            when 'v' then format('view:%I.%I', ns.nspname, c.relname)
+            when 'm' then format('materializedView:%I.%I', ns.nspname, c.relname)
+            when 'S' then format('sequence:%I.%I', ns.nspname, c.relname)
+            when 'i' then format('index:%I.%I.%I', ns.nspname, tbl.relname, c.relname)
+            when 'c' then format('type:%I.%I', ns.nspname, c.relname)
+            else format('unknown:%s.%s', 'pg_class', c.oid::text)
+          end as stable_id
+    from pg_class c
+    join pg_namespace ns on ns.oid = c.relnamespace
+    left join pg_index idx on idx.indexrelid = c.oid
+    left join pg_class tbl on tbl.oid = idx.indrelid
+    join ids i on i.classid = 'pg_class'::regclass and i.objid = c.oid and i.objsubid = 0
+
+    union all
+
+    select 'pg_class'::regclass, a.attrelid, a.attnum,
+          format('column:%I.%I.%I', ns.nspname, c.relname, a.attname) as stable_id
+    from pg_attribute a
+    join pg_class c on c.oid = a.attrelid
+    join pg_namespace ns on ns.oid = c.relnamespace
+    join ids i on i.classid = 'pg_class'::regclass and i.objid = a.attrelid and i.objsubid = a.attnum
+    where a.attnum > 0 and not a.attisdropped
+
+    union all
+
+    select 'pg_type'::regclass, t.oid, 0::int2,
+          case t.typtype
+            when 'd' then format('domain:%I.%I', ns.nspname, t.typname)
+            when 'e' then format('type:%I.%I', ns.nspname, t.typname)
+            when 'r' then format('type:%I.%I', ns.nspname, t.typname)
+            when 'm' then format('multirange:%I.%I', ns.nspname, t.typname)
+            when 'c' then
+              case
+                when r.oid is not null and r.relkind in ('r','p','f') then format('table:%I.%I', rns.nspname, r.relname)
+                when r.oid is not null and r.relkind = 'v' then format('view:%I.%I', rns.nspname, r.relname)
+                when r.oid is not null and r.relkind = 'm' then format('materializedView:%I.%I', rns.nspname, r.relname)
+                else format('type:%I.%I', ns.nspname, t.typname)
+              end
+            when 'p' then format('pseudoType:%I.%I', ns.nspname, t.typname)
+            else format('type:%I.%I', ns.nspname, t.typname)
+          end as stable_id
+    from pg_type t
+    join pg_namespace ns on ns.oid = t.typnamespace
+    left join pg_class r on r.oid = t.typrelid
+    left join pg_namespace rns on rns.oid = r.relnamespace
+    join ids i on i.classid = 'pg_type'::regclass and i.objid = t.oid and i.objsubid = 0
+
+    union all
+
+    select 'pg_constraint'::regclass, c.oid, 0::int2,
+          case
+            when c.contypid <> 0 then format('constraint:%I.%I.%I', ns.nspname, ty.typname, c.conname)
+            when c.conrelid <> 0 then format('constraint:%I.%I.%I', tbl_ns.nspname, tbl.relname, c.conname)
+            else format('constraint:%s', c.oid::text)
+          end as stable_id
+    from pg_constraint c
+    left join pg_type ty on ty.oid = c.contypid
+    left join pg_namespace ns on ns.oid = ty.typnamespace
+    left join pg_class tbl on tbl.oid = c.conrelid
+    left join pg_namespace tbl_ns on tbl_ns.oid = tbl.relnamespace
+    join ids i on i.classid = 'pg_constraint'::regclass and i.objid = c.oid and i.objsubid = 0
+
+    union all
+
+    select 'pg_proc'::regclass, p.oid, 0::int2,
+          format(
+            'procedure:%I.%I(%s)',
+            ns.nspname,
+            p.proname,
+            coalesce((select string_agg(format_type(oid, null), ',' order by ord)
+              from unnest(p.proargtypes) with ordinality as t(oid, ord)), '')
+          ) as stable_id
+    from pg_proc p
+    join pg_namespace ns on ns.oid = p.pronamespace
+    join ids i on i.classid = 'pg_proc'::regclass and i.objid = p.oid and i.objsubid = 0
+
+    union all
+
+    select 'pg_trigger'::regclass, tg.oid, 0::int2,
+          format('trigger:%I.%I.%I', ns.nspname, tbl.relname, tg.tgname) as stable_id
+    from pg_trigger tg
+    join pg_class tbl on tbl.oid = tg.tgrelid
+    join pg_namespace ns on ns.oid = tbl.relnamespace
+    join ids i on i.classid = 'pg_trigger'::regclass and i.objid = tg.oid and i.objsubid = 0
+
+    union all
+
+    select 'pg_collation'::regclass, c.oid, 0::int2,
+          format('collation:%I.%I', ns.nspname, c.collname) as stable_id
+    from pg_collation c
+    join pg_namespace ns on ns.oid = c.collnamespace
+    join ids i on i.classid = 'pg_collation'::regclass and i.objid = c.oid and i.objsubid = 0
+
+    union all
+
+    select 'pg_event_trigger'::regclass, et.oid, 0::int2,
+          format('eventTrigger:%I', et.evtname) as stable_id
+    from pg_event_trigger et
+    join ids i on i.classid = 'pg_event_trigger'::regclass and i.objid = et.oid and i.objsubid = 0
+
+    union all
+
+    select 'pg_ts_config'::regclass, cfg.oid, 0::int2,
+          format('tsConfig:%I.%I', ns.nspname, cfg.cfgname) as stable_id
+    from pg_ts_config cfg
+    join pg_namespace ns on ns.oid = cfg.cfgnamespace
+    join ids i on i.classid = 'pg_ts_config'::regclass and i.objid = cfg.oid and i.objsubid = 0
+
+    union all
+
+    select 'pg_ts_dict'::regclass, dict.oid, 0::int2,
+          format('tsDict:%I.%I', ns.nspname, dict.dictname) as stable_id
+    from pg_ts_dict dict
+    join pg_namespace ns on ns.oid = dict.dictnamespace
+    join ids i on i.classid = 'pg_ts_dict'::regclass and i.objid = dict.oid and i.objsubid = 0
+
+    union all
+
+    select 'pg_ts_template'::regclass, tmpl.oid, 0::int2,
+          format('tsTemplate:%I.%I', ns.nspname, tmpl.tmplname) as stable_id
+    from pg_ts_template tmpl
+    join pg_namespace ns on ns.oid = tmpl.tmplnamespace
+    join ids i on i.classid = 'pg_ts_template'::regclass and i.objid = tmpl.oid and i.objsubid = 0
+  ), extension_members as (
+    select
+      em.extension_oid,
+      obj.stable_id
+    from extension_members_raw em
+    join objects obj
+      on obj.classid = em.classid
+      and obj.objid = em.objid
+      and obj.objsubid = coalesce(nullif(em.objsubid, 0), 0)
+  )
+  select
+    er.name,
+    er.schema,
+    er.relocatable,
+    er.version,
+    er.owner,
+    er.comment,
+    coalesce(
+      (
+        select json_agg(em.stable_id order by em.stable_id)
+        from extension_members em
+        where em.extension_oid = er.oid
+      ), '[]'::json
+    ) as members
+  from extension_rows er
+  order by
+    er.name
+  `);
+  // Validate and parse each row using the Zod schema
+  const validatedRows = extensionRows.map((row: unknown) =>
+    extensionPropsSchema.parse(row),
+  );
+  return validatedRows.map((row: ExtensionProps) => new Extension(row));
+}

package/src/core/objects/extract-with-retry.test.ts

@@ -0,0 +1,143 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import {
+  extractWithDefinitionRetry,
+  resolveExtractRetries,
+} from "./extract-with-retry.ts";
+
+type Row = { id: string; definition: string | null };
+
+const hasNullDefinition = (r: Row) => r.definition === null;
+
+describe("resolveExtractRetries", () => {
+  const originalEnv = process.env.PGDELTA_EXTRACT_RETRIES;
+  afterEach(() => {
+    if (originalEnv === undefined) {
+      process.env.PGDELTA_EXTRACT_RETRIES = undefined;
+      delete process.env.PGDELTA_EXTRACT_RETRIES;
+    } else {
+      process.env.PGDELTA_EXTRACT_RETRIES = originalEnv;
+    }
+  });
+
+  test("defaults to 1 when option and env are unset", () => {
+    delete process.env.PGDELTA_EXTRACT_RETRIES;
+    expect(resolveExtractRetries()).toBe(1);
+  });
+
+  test("uses option when provided", () => {
+    process.env.PGDELTA_EXTRACT_RETRIES = "5";
+    expect(resolveExtractRetries(0)).toBe(0);
+    expect(resolveExtractRetries(1)).toBe(1);
+    expect(resolveExtractRetries(7)).toBe(7);
+  });
+
+  test("falls back to env when option is undefined", () => {
+    process.env.PGDELTA_EXTRACT_RETRIES = "4";
+    expect(resolveExtractRetries()).toBe(4);
+  });
+
+  test("clamps negative values to 0", () => {
+    delete process.env.PGDELTA_EXTRACT_RETRIES;
+    expect(resolveExtractRetries(-3)).toBe(0);
+    process.env.PGDELTA_EXTRACT_RETRIES = "-9";
+    expect(resolveExtractRetries()).toBe(0);
+  });
+
+  test("ignores non-numeric env values", () => {
+    process.env.PGDELTA_EXTRACT_RETRIES = "not-a-number";
+    expect(resolveExtractRetries()).toBe(1);
+  });
+
+  test("ignores empty env string", () => {
+    process.env.PGDELTA_EXTRACT_RETRIES = "";
+    expect(resolveExtractRetries()).toBe(1);
+  });
+});
+
+describe("extractWithDefinitionRetry", () => {
+  test("returns first attempt when no row has null definition", async () => {
+    let attempts = 0;
+    const rows = await extractWithDefinitionRetry<Row>({
+      label: "test",
+      query: async () => {
+        attempts++;
+        return [{ id: "a", definition: "OK" }];
+      },
+      hasNullDefinition,
+      options: { retries: 2, backoffMs: 0 },
+    });
+    expect(attempts).toBe(1);
+    expect(rows).toEqual([{ id: "a", definition: "OK" }]);
+  });
+
+  test("retries when definition is null and succeeds on attempt 2", async () => {
+    let attempts = 0;
+    const rows = await extractWithDefinitionRetry<Row>({
+      label: "test",
+      query: async () => {
+        attempts++;
+        if (attempts === 1) {
+          return [
+            { id: "a", definition: "OK" },
+            { id: "b", definition: null },
+          ];
+        }
+        return [{ id: "a", definition: "OK" }];
+      },
+      hasNullDefinition,
+      options: { retries: 2, backoffMs: 0 },
+    });
+    expect(attempts).toBe(2);
+    expect(rows).toEqual([{ id: "a", definition: "OK" }]);
+  });
+
+  test("returns last-attempt rows (with offenders) once retries are exhausted", async () => {
+    let attempts = 0;
+    const rows = await extractWithDefinitionRetry<Row>({
+      label: "test",
+      query: async () => {
+        attempts++;
+        return [
+          { id: "a", definition: "OK" },
+          { id: "b", definition: null },
+        ];
+      },
+      hasNullDefinition,
+      options: { retries: 2, backoffMs: 0 },
+    });
+    expect(attempts).toBe(3);
+    expect(rows).toEqual([
+      { id: "a", definition: "OK" },
+      { id: "b", definition: null },
+    ]);
+  });
+
+  test("retries: 0 disables retrying entirely", async () => {
+    let attempts = 0;
+    const rows = await extractWithDefinitionRetry<Row>({
+      label: "test",
+      query: async () => {
+        attempts++;
+        return [{ id: "b", definition: null }];
+      },
+      hasNullDefinition,
+      options: { retries: 0, backoffMs: 0 },
+    });
+    expect(attempts).toBe(1);
+    expect(rows).toEqual([{ id: "b", definition: null }]);
+  });
+
+  test("retries: 5 attempts up to 6 times before giving up", async () => {
+    let attempts = 0;
+    await extractWithDefinitionRetry<Row>({
+      label: "test",
+      query: async () => {
+        attempts++;
+        return [{ id: "b", definition: null }];
+      },
+      hasNullDefinition,
+      options: { retries: 5, backoffMs: 0 },
+    });
+    expect(attempts).toBe(6);
+  });
+});

package/src/core/objects/extract-with-retry.ts

@@ -0,0 +1,87 @@
+import debug from "debug";
+
+const log = debug("pg-delta:extract");
+
+const DEFAULT_RETRIES = 1;
+const DEFAULT_BACKOFF_MS = 50;
+
+export interface ExtractRetryOptions {
+  /**
+   * Number of retry attempts to make when a `pg_get_*def()` call returns NULL
+   * for at least one row. Total attempts is `retries + 1`. Negative values are
+   * clamped to 0. When this option is undefined the value is read from the
+   * `PGDELTA_EXTRACT_RETRIES` environment variable, falling back to a default
+   * of 1 (i.e. the first attempt plus one retry, 2 attempts total).
+   */
+  retries?: number;
+  /**
+   * Delay between retry attempts in milliseconds; the actual wait is
+   * `backoffMs * attemptNumber` (linear). Defaults to 50. Set to 0 in tests.
+   */
+  backoffMs?: number;
+}
+
+export function resolveExtractRetries(option?: number): number {
+  if (typeof option === "number" && Number.isFinite(option)) {
+    return Math.max(0, Math.floor(option));
+  }
+  const envVal = process.env.PGDELTA_EXTRACT_RETRIES;
+  if (envVal !== undefined && envVal !== "") {
+    const n = Number(envVal);
+    if (Number.isFinite(n)) return Math.max(0, Math.floor(n));
+  }
+  return DEFAULT_RETRIES;
+}
+
+const sleep = (ms: number) =>
+  ms > 0 ? new Promise<void>((r) => setTimeout(r, ms)) : Promise.resolve();
+
+/**
+ * Runs `query()` up to `retries + 1` times, retrying as long as at least one
+ * row in the result satisfies `hasNullDefinition`. The retry exists because
+ * `pg_get_<x>def()` can return NULL transiently when the underlying catalog
+ * row is dropped concurrently or the catalog state is in flux; in practice a
+ * second attempt either no longer sees the dropped row or succeeds in
+ * resolving the definition.
+ *
+ * Returns the rows from the first attempt with no offenders, or — once
+ * retries are exhausted — the rows from the final attempt (still containing
+ * offenders). The caller is responsible for the final filter so this helper
+ * works for both flat schemas (definition on the row) and nested schemas
+ * (definition on a child collection, e.g. table constraints).
+ */
+export async function extractWithDefinitionRetry<TRow>(params: {
+  label: string;
+  query: () => Promise<TRow[]>;
+  hasNullDefinition: (row: TRow) => boolean;
+  options?: ExtractRetryOptions;
+}): Promise<TRow[]> {
+  const retries = resolveExtractRetries(params.options?.retries);
+  const backoffMs = params.options?.backoffMs ?? DEFAULT_BACKOFF_MS;
+  const maxAttempts = retries + 1;
+
+  let rows: TRow[] = [];
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    rows = await params.query();
+    const offenders = rows.filter(params.hasNullDefinition).length;
+    if (offenders === 0) return rows;
+    if (attempt < maxAttempts) {
+      log(
+        "%s: pg_get_*def() returned NULL for %d row(s) on attempt %d/%d; retrying",
+        params.label,
+        offenders,
+        attempt,
+        maxAttempts,
+      );
+      await sleep(backoffMs * attempt);
+    } else {
+      log(
+        "%s: pg_get_*def() returned NULL for %d row(s) after %d attempt(s); skipping",
+        params.label,
+        offenders,
+        maxAttempts,
+      );
+    }
+  }
+  return rows;
+}