@supabase/pg-delta 1.0.0-alpha.1 → 1.0.0-alpha.11

package/src/core/declarative-apply/discover-sql.ts

@@ -0,0 +1,107 @@
+/**
+ * Discover and read .sql files under a schema path (file or directory).
+ * Matches pg-topo's discovery order for deterministic statement ordering.
+ */
+
+import { readdir, readFile, stat } from "node:fs/promises";
+import path from "node:path";
+
+export interface SqlFileEntry {
+  /** Relative path from base (forward slashes, e.g. schemas/public/views/billing.sql) */
+  filePath: string;
+  /** File content */
+  sql: string;
+}
+
+/**
+ * Recursively collect .sql files in a directory. Entries sorted by name,
+ * then full paths sorted for deterministic order (matches pg-topo discover).
+ */
+async function readSqlFilesInDirectory(
+  directoryPath: string,
+  outFiles: Set<string>,
+): Promise<void> {
+  const entries = await readdir(directoryPath, { withFileTypes: true });
+  entries.sort((left, right) => left.name.localeCompare(right.name));
+
+  for (const entry of entries) {
+    const fullPath = path.join(directoryPath, entry.name);
+    if (entry.isDirectory()) {
+      await readSqlFilesInDirectory(fullPath, outFiles);
+      continue;
+    }
+
+    if (entry.isFile() && fullPath.toLowerCase().endsWith(".sql")) {
+      outFiles.add(path.resolve(fullPath));
+    }
+  }
+}
+
+/**
+ * Stable relative path: path.relative(basePath, absolutePath) with forward slashes.
+ */
+function toStablePath(absolutePath: string, basePath: string): string {
+  return path.relative(basePath, absolutePath).split(path.sep).join("/");
+}
+
+/**
+ * Load all .sql files under schemaPath (a single .sql file or a directory).
+ * Returns entries in the same order as pg-topo's discover (sorted by full path).
+ *
+ * @throws If schemaPath does not exist, is not a file/directory, or any file cannot be read.
+ *         Error message includes path and code (e.g. ENOENT, EACCES) for CLI to display.
+ */
+export async function loadDeclarativeSchema(
+  schemaPath: string,
+): Promise<SqlFileEntry[]> {
+  const resolvedRoot = path.resolve(schemaPath);
+
+  let rootStats: Awaited<ReturnType<typeof stat>>;
+  try {
+    rootStats = await stat(resolvedRoot);
+  } catch (err) {
+    const code =
+      err && typeof err === "object" && "code" in err
+        ? String((err as NodeJS.ErrnoException).code)
+        : "UNKNOWN";
+    throw new Error(`Cannot access '${schemaPath}': ${code}`);
+  }
+
+  let files: string[];
+  let basePath: string;
+
+  if (rootStats.isFile()) {
+    if (!resolvedRoot.toLowerCase().endsWith(".sql")) {
+      throw new Error(`Path is not a .sql file: '${schemaPath}'`);
+    }
+    files = [resolvedRoot];
+    basePath = path.dirname(resolvedRoot);
+  } else if (rootStats.isDirectory()) {
+    const fileSet = new Set<string>();
+    await readSqlFilesInDirectory(resolvedRoot, fileSet);
+    files = [...fileSet].sort((a, b) => a.localeCompare(b));
+    basePath = resolvedRoot;
+  } else {
+    throw new Error(`Path is not a file or directory: '${schemaPath}'`);
+  }
+
+  const entries: SqlFileEntry[] = [];
+  for (const filePath of files) {
+    try {
+      const sql = await readFile(filePath, "utf-8");
+      entries.push({
+        filePath: toStablePath(filePath, basePath),
+        sql,
+      });
+    } catch (err) {
+      const code =
+        err && typeof err === "object" && "code" in err
+          ? String((err as NodeJS.ErrnoException).code)
+          : "UNKNOWN";
+      const relative = toStablePath(filePath, basePath);
+      throw new Error(`Cannot read file '${relative}': ${code}`);
+    }
+  }
+
+  return entries;
+}

package/src/core/declarative-apply/extract-catalog-providers.ts

@@ -0,0 +1,220 @@
+/**
+ * Extract functions, types, and other catalog objects from the target database
+ * so pg-topo can treat them as external providers and suppress false
+ * UNRESOLVED_DEPENDENCY diagnostics (e.g. now(), gen_random_uuid(), nextval(),
+ * auth.users, extensions.uuid_generate_v4, etc.).
+ */
+
+import type { ObjectRef } from "@supabase/pg-topo";
+import type { Pool } from "pg";
+
+type FunctionRow = {
+  name: string;
+  schema: string;
+  kind: string;
+  signature: string;
+};
+type TypeRow = { name: string; schema: string; typetype: string };
+type SchemaRow = { name: string };
+type RelationRow = { name: string; schema: string; relkind: string };
+type ExtensionRow = { name: string; schema: string | null };
+type RoleRow = { name: string };
+type LanguageRow = { name: string };
+type CollationRow = { name: string; schema: string };
+type FdwRow = { name: string };
+type ServerRow = { name: string };
+type EventTriggerRow = { name: string };
+type PublicationRow = { name: string };
+type SubscriptionRow = { name: string };
+
+function addProvider(
+  providers: ObjectRef[],
+  ref: ObjectRef,
+  alsoUnderPublic = false,
+): void {
+  providers.push(ref);
+  if (
+    alsoUnderPublic &&
+    (ref.schema === "pg_catalog" || ref.schema === "information_schema")
+  ) {
+    providers.push({ ...ref, schema: "public" });
+  }
+}
+
+/**
+ * Query the target database for all catalog objects that can be dependencies.
+ * Returns ObjectRefs that pg-topo can use as external providers so it does
+ * not flag false UNRESOLVED_DEPENDENCY diagnostics (e.g. now(), text, public).
+ *
+ * This is intentionally separate from {@link extractCatalog} in catalog.model.ts:
+ * - extractCatalog is for schema diffing and excludes system objects (filters
+ *   out pg_catalog, information_schema, extension-owned). It also does not
+ *   extract languages. We need the opposite here: all objects that might be
+ *   referenced by SQL, including built-in functions, types, and schemas.
+ * - Objects in pg_catalog/information_schema are registered under both their
+ *   real schema and "public" (via addProvider's alsoUnderPublic) so
+ *   unqualified references in SQL resolve the same way the parser does.
+ */
+export async function extractCatalogProviders(
+  pool: Pool,
+): Promise<ObjectRef[]> {
+  const providers: ObjectRef[] = [];
+
+  const [
+    functionsResult,
+    typesResult,
+    schemasResult,
+    relationsResult,
+    extensionsResult,
+    rolesResult,
+    languagesResult,
+    collationsResult,
+    fdwsResult,
+    serversResult,
+    eventTriggersResult,
+    publicationsResult,
+    subscriptionsResult,
+  ] = await Promise.all([
+    pool.query<FunctionRow>(`
+      SELECT
+        p.proname AS name,
+        n.nspname AS schema,
+        p.prokind AS kind,
+        COALESCE((
+          SELECT string_agg(
+            CASE WHEN p.proargmodes IS NOT NULL AND p.proargmodes[ord] = 'v'
+                 THEN 'VARIADIC ' || format_type(t.oid, NULL)
+                 ELSE format_type(t.oid, NULL)
+            END, ',' ORDER BY ord)
+          FROM unnest(p.proargtypes) WITH ORDINALITY AS t(oid, ord)
+        ), '') AS signature
+      FROM pg_proc p
+      JOIN pg_namespace n ON n.oid = p.pronamespace
+    `),
+    pool.query<TypeRow>(`
+      SELECT t.typname AS name, n.nspname AS schema, t.typtype AS typetype
+      FROM pg_type t
+      JOIN pg_namespace n ON n.oid = t.typnamespace
+      WHERE n.nspname NOT LIKE 'pg_toast%'
+        AND t.typtype IN ('b', 'c', 'd', 'e', 'r')
+    `),
+    pool.query<SchemaRow>(`
+      SELECT nspname AS name FROM pg_namespace
+      WHERE nspname NOT LIKE 'pg_toast%'
+    `),
+    pool.query<RelationRow>(`
+      SELECT c.relname AS name, n.nspname AS schema, c.relkind AS relkind
+      FROM pg_class c
+      JOIN pg_namespace n ON n.oid = c.relnamespace
+      WHERE c.relkind IN ('r', 'p', 'v', 'm', 'S', 'i')
+        AND n.nspname NOT LIKE 'pg_toast%'
+    `),
+    pool.query<ExtensionRow>(`
+      SELECT e.extname AS name, n.nspname AS schema
+      FROM pg_extension e
+      LEFT JOIN pg_namespace n ON n.oid = e.extnamespace
+    `),
+    pool.query<RoleRow>(`SELECT rolname AS name FROM pg_roles`),
+    pool.query<LanguageRow>(`SELECT lanname AS name FROM pg_language`),
+    pool.query<CollationRow>(`
+      SELECT c.collname AS name, n.nspname AS schema
+      FROM pg_collation c
+      JOIN pg_namespace n ON n.oid = c.collnamespace
+      WHERE n.nspname NOT LIKE 'pg_toast%'
+    `),
+    pool.query<FdwRow>(`SELECT fdwname AS name FROM pg_foreign_data_wrapper`),
+    pool.query<ServerRow>(`SELECT srvname AS name FROM pg_foreign_server`),
+    pool.query<EventTriggerRow>(`SELECT evtname AS name FROM pg_event_trigger`),
+    pool.query<PublicationRow>(`SELECT pubname AS name FROM pg_publication`),
+    pool.query<SubscriptionRow>(`SELECT subname AS name FROM pg_subscription`),
+  ]);
+
+  for (const fn of functionsResult.rows) {
+    const kind =
+      fn.kind === "a"
+        ? "aggregate"
+        : fn.kind === "p"
+          ? "procedure"
+          : "function";
+    const sig = fn.signature.trim() ? `(${fn.signature})` : "()";
+    const ref: ObjectRef = {
+      kind,
+      name: fn.name,
+      schema: fn.schema,
+      signature: sig,
+    };
+    addProvider(providers, ref, true);
+  }
+
+  for (const t of typesResult.rows) {
+    const kind = t.typetype === "d" ? "domain" : "type";
+    const ref: ObjectRef = { kind, name: t.name, schema: t.schema };
+    addProvider(providers, ref, true);
+  }
+
+  for (const row of schemasResult.rows) {
+    providers.push({ kind: "schema", name: row.name });
+  }
+
+  const relkindToKind: Record<string, ObjectRef["kind"]> = {
+    r: "table",
+    p: "table",
+    v: "view",
+    m: "materialized_view",
+    S: "sequence",
+    i: "index",
+  };
+  for (const row of relationsResult.rows) {
+    const kind = relkindToKind[row.relkind];
+    if (!kind) continue;
+    const ref: ObjectRef = { kind, name: row.name, schema: row.schema };
+    addProvider(providers, ref, true);
+  }
+
+  for (const row of extensionsResult.rows) {
+    providers.push({
+      kind: "extension",
+      name: row.name,
+      schema: row.schema ?? undefined,
+    });
+  }
+
+  for (const row of rolesResult.rows) {
+    providers.push({ kind: "role", name: row.name });
+  }
+
+  for (const row of languagesResult.rows) {
+    providers.push({ kind: "language", name: row.name });
+  }
+
+  for (const row of collationsResult.rows) {
+    const ref: ObjectRef = {
+      kind: "collation",
+      name: row.name,
+      schema: row.schema,
+    };
+    addProvider(providers, ref, true);
+  }
+
+  for (const row of fdwsResult.rows) {
+    providers.push({ kind: "foreign_data_wrapper", name: row.name });
+  }
+
+  for (const row of serversResult.rows) {
+    providers.push({ kind: "foreign_server", name: row.name });
+  }
+
+  for (const row of eventTriggersResult.rows) {
+    providers.push({ kind: "event_trigger", name: row.name });
+  }
+
+  for (const row of publicationsResult.rows) {
+    providers.push({ kind: "publication", name: row.name });
+  }
+
+  for (const row of subscriptionsResult.rows) {
+    providers.push({ kind: "subscription", name: row.name });
+  }
+
+  return providers;
+}

package/src/core/declarative-apply/index.test.ts

@@ -0,0 +1,67 @@
+import { describe, expect, mock, test } from "bun:test";
+import type { Pool } from "pg";
+import { applyDeclarativeSchema } from "./index.ts";
+
+// Mock extractCatalogProviders to throw, simulating an early failure
+// before roundApply is ever reached.
+mock.module("./extract-catalog-providers.ts", () => ({
+  extractCatalogProviders: async () => {
+    throw new Error("simulated catalog extraction failure");
+  },
+}));
+
+// Track the pool created internally via createManagedPool so we can verify cleanup.
+let lastCreatedPool: Pool & { closeCalled: boolean };
+
+mock.module("../postgres-config.ts", () => ({
+  createManagedPool: async () => {
+    lastCreatedPool = createMockPool();
+    return {
+      pool: lastCreatedPool,
+      close: async () => {
+        lastCreatedPool.closeCalled = true;
+      },
+    };
+  },
+}));
+
+function createMockPool(): Pool & { closeCalled: boolean } {
+  const pool = {
+    closeCalled: false,
+    connect: async () => {
+      throw new Error("should not connect");
+    },
+    end: async () => {},
+    query: async () => {
+      throw new Error("should not query");
+    },
+  } as unknown as Pool & { closeCalled: boolean };
+  return pool;
+}
+
+describe("applyDeclarativeSchema", () => {
+  test("caller-owned pool is NOT closed on early failure", async () => {
+    const pool = createMockPool();
+
+    await expect(
+      applyDeclarativeSchema({
+        content: [{ filePath: "test.sql", sql: "CREATE TABLE t(id int);" }],
+        pool,
+      }),
+    ).rejects.toThrow("simulated catalog extraction failure");
+
+    expect(pool.closeCalled).toBe(false);
+  });
+
+  test("internally-created pool IS closed on early failure", async () => {
+    await expect(
+      applyDeclarativeSchema({
+        content: [{ filePath: "test.sql", sql: "CREATE TABLE t(id int);" }],
+        targetUrl: "postgresql://localhost/test",
+      }),
+    ).rejects.toThrow("simulated catalog extraction failure");
+
+    expect(lastCreatedPool).toBeDefined();
+    expect(lastCreatedPool.closeCalled).toBe(true);
+  });
+});

package/src/core/declarative-apply/index.ts

@@ -0,0 +1,205 @@
+/**
+ * Declarative schema apply – orchestrator.
+ *
+ * Accepts pre-read SQL content (file path + sql string per file), uses pg-topo
+ * for static dependency analysis and topological ordering, then applies
+ * statements to a target database using iterative rounds to handle any
+ * remaining dependency gaps. File discovery and reading are done by the caller
+ * (e.g. CLI) so I/O errors can be handled there.
+ */
+
+import type { Diagnostic, StatementNode } from "@supabase/pg-topo";
+import { analyzeAndSort } from "@supabase/pg-topo";
+import type { Pool } from "pg";
+import { createManagedPool } from "../postgres-config.ts";
+import { extractCatalogProviders } from "./extract-catalog-providers.ts";
+import {
+  type ApplyResult,
+  type RoundResult,
+  roundApply,
+  type StatementEntry,
+} from "./round-apply.ts";
+
+// ---------------------------------------------------------------------------
+// Public types
+// ---------------------------------------------------------------------------
+
+import type { SqlFileEntry } from "./discover-sql.ts";
+
+interface DeclarativeApplyOptions {
+  /** Pre-read SQL files: filePath (relative) and sql content. Caller does discovery and read. */
+  content: SqlFileEntry[];
+  /** Target database connection URL (required if pool is not provided) */
+  targetUrl?: string;
+  /** Existing pool to use (caller owns it; not closed). If provided, targetUrl is ignored. */
+  pool?: Pool;
+  /** Max rounds before giving up (default: 100) */
+  maxRounds?: number;
+  /** Run final function body validation (default: true) */
+  validateFunctionBodies?: boolean;
+  /** Disable function body checks during rounds (default: true) */
+  disableCheckFunctionBodies?: boolean;
+  /** Progress callback fired after each round */
+  onRoundComplete?: (round: RoundResult) => void;
+}
+
+export interface DeclarativeApplyResult {
+  /** Result from the round-based apply engine */
+  apply: ApplyResult;
+  /** Diagnostics from pg-topo's static analysis (warnings, not fatal) */
+  diagnostics: Diagnostic[];
+  /** Total number of statements discovered */
+  totalStatements: number;
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Convert pg-topo StatementNodes into StatementEntries for the apply engine.
+ */
+function toStatementEntries(nodes: StatementNode[]): StatementEntry[] {
+  return nodes.map((node) => ({
+    id: `${node.id.filePath}:${node.id.statementIndex}`,
+    sql: node.sql,
+    statementClass: node.statementClass,
+  }));
+}
+
+function remapStatementId(
+  statementId: { filePath: string; statementIndex: number } | undefined,
+  filePathMap: Map<string, string>,
+): typeof statementId {
+  if (!statementId) return undefined;
+  return {
+    ...statementId,
+    filePath: filePathMap.get(statementId.filePath) ?? statementId.filePath,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Main entry point
+// ---------------------------------------------------------------------------
+
+/**
+ * Apply a declarative SQL schema to a target database.
+ *
+ * 1. Call pg-topo analyzeAndSort on the provided SQL strings
+ * 2. Remap synthetic statement IDs to caller-provided file paths
+ * 3. Apply statements round-by-round to the target database
+ * 4. Optionally validate function bodies in a final pass
+ */
+export async function applyDeclarativeSchema(
+  options: DeclarativeApplyOptions,
+): Promise<DeclarativeApplyResult> {
+  const {
+    content,
+    targetUrl,
+    pool: providedPool,
+    maxRounds = 100,
+    validateFunctionBodies = true,
+    disableCheckFunctionBodies = true,
+    onRoundComplete,
+  } = options;
+
+  if (content.length === 0) {
+    return {
+      apply: {
+        status: "success",
+        totalRounds: 0,
+        totalApplied: 0,
+        totalSkipped: 0,
+        rounds: [],
+      },
+      diagnostics: [],
+      totalStatements: 0,
+    };
+  }
+
+  let pool: Pool;
+  let closePool: (() => Promise<void>) | undefined;
+  if (providedPool != null) {
+    pool = providedPool;
+  } else if (targetUrl != null) {
+    const managed = await createManagedPool(targetUrl, { label: "target" });
+    pool = managed.pool;
+    closePool = managed.close;
+  } else {
+    throw new Error("Either targetUrl or pool must be provided");
+  }
+
+  try {
+    const externalProviders = await extractCatalogProviders(pool);
+
+    // Step 1: pg-topo analyze and sort (no file I/O; uses synthetic <input:i> paths)
+    const sqlContents = content.map((entry) => entry.sql);
+    const analyzeResult = await analyzeAndSort(sqlContents, {
+      externalProviders,
+    });
+
+    const { ordered, diagnostics } = analyzeResult;
+
+    // Step 2: Remap <input:i> to real file paths
+    const filePathMap = new Map<string, string>();
+    for (let i = 0; i < content.length; i += 1) {
+      filePathMap.set(`<input:${i}>`, content[i].filePath);
+    }
+
+    const remappedOrdered = ordered.map((node) => ({
+      ...node,
+      id: {
+        ...node.id,
+        filePath: filePathMap.get(node.id.filePath) ?? node.id.filePath,
+      },
+    }));
+
+    const remappedDiagnostics = diagnostics.map((d) => ({
+      ...d,
+      statementId: remapStatementId(d.statementId, filePathMap),
+    }));
+
+    if (ordered.length === 0) {
+      return {
+        apply: {
+          status: "success",
+          totalRounds: 0,
+          totalApplied: 0,
+          totalSkipped: 0,
+          rounds: [],
+        },
+        diagnostics: remappedDiagnostics,
+        totalStatements: 0,
+      };
+    }
+
+    // Step 3: Convert to statement entries and apply
+    const statements = toStatementEntries(remappedOrdered);
+
+    const applyResult = await roundApply({
+      pool,
+      statements,
+      maxRounds,
+      disableCheckFunctionBodies,
+      finalValidation: validateFunctionBodies,
+      onRoundComplete,
+    });
+
+    return {
+      apply: applyResult,
+      diagnostics: remappedDiagnostics,
+      totalStatements: remappedOrdered.length,
+    };
+  } finally {
+    if (closePool) {
+      await closePool();
+    }
+  }
+}
+
+export type { SqlFileEntry } from "./discover-sql.ts";
+
+// Re-export file discovery for programmatic callers (e.g. Supabase CLI edge-runtime templates)
+export { loadDeclarativeSchema } from "./discover-sql.ts";
+// Re-export result types for callers that need them (StatementError is imported from round-apply directly where needed)
+export type { ApplyResult, RoundResult } from "./round-apply.ts";