@supabase/pg-delta 1.0.0-alpha.1 → 1.0.0-alpha.11

package/dist/core/declarative-apply/round-apply.js

@@ -0,0 +1,378 @@
+/**
+ * Round-based declarative schema apply engine.
+ *
+ * Applies SQL statements to a database using iterative rounds:
+ * 1. Try each pending statement
+ * 2. On dependency errors, defer to next round
+ * 3. Repeat until all applied or no progress (stuck)
+ * 4. Optional final validation pass for function bodies
+ */
+import debug from "debug";
+const debugApply = debug("pg-delta:declarative-apply");
+// ---------------------------------------------------------------------------
+// Dependency error classification
+// ---------------------------------------------------------------------------
+/**
+ * SQLSTATE codes that indicate a missing dependency (object not yet created).
+ * Mirrors pg-topo's isDependencyErrorCode.
+ */
+const DEPENDENCY_ERROR_CODES = new Set([
+    "42P01", // undefined_table
+    "42703", // undefined_column
+    "42704", // undefined_object
+    "42883", // undefined_function
+    "3F000", // invalid_schema_name
+]);
+/**
+ * Detect errors caused by environment/capability limitations rather than
+ * schema bugs. These statements are skipped permanently (not retried).
+ *
+ * Strategy: SQLSTATE codes are the primary gate (fast, stable). For codes
+ * reused across unrelated error conditions (e.g. 42710 = "duplicate object"
+ * covers both roles and extensions), the error message is used as a secondary
+ * disambiguator. Messages are lowercased before matching to handle
+ * case-sensitivity differences across PG versions.
+ */
+function isEnvironmentCapabilityError(code, message, statementClass) {
+    // Feature not supported
+    if (code === "0A000")
+        return true;
+    // Extension not available
+    if (code === "58P01" &&
+        message.includes("extension") &&
+        (message.includes("control file") || message.includes("is not available"))) {
+        return true;
+    }
+    // Subscription / logical replication not available
+    if (statementClass === "CREATE_SUBSCRIPTION" &&
+        (code === "58P01" ||
+            message.includes("walreceiver") ||
+            message.includes("logical replication"))) {
+        return true;
+    }
+    // Event trigger requires superuser
+    if (statementClass === "CREATE_EVENT_TRIGGER" &&
+        (code === "42501" || message.includes("must be superuser"))) {
+        return true;
+    }
+    // Language does not exist (e.g. plv8)
+    if ((statementClass === "CREATE_FUNCTION" ||
+        statementClass === "CREATE_PROCEDURE") &&
+        message.includes("language") &&
+        message.includes("does not exist")) {
+        return true;
+    }
+    // Role already exists
+    if (statementClass === "CREATE_ROLE" &&
+        (code === "42710" || code === "23505") &&
+        message.includes("role") &&
+        (message.includes("already exists") ||
+            message.includes("duplicate key") ||
+            message.includes("pg_authid_rolname_index"))) {
+        return true;
+    }
+    // Extension already exists
+    if (statementClass === "CREATE_EXTENSION" &&
+        code === "42710" &&
+        message.includes("extension") &&
+        message.includes("already exists")) {
+        return true;
+    }
+    // Sequence ownership constraint
+    if (code === "55000" &&
+        message.includes("sequence must have same owner as table it is linked to")) {
+        return true;
+    }
+    // Publication replica identity
+    if (code === "55000" &&
+        message.includes("does not have a replica identity") &&
+        message.includes("publishes updates")) {
+        return true;
+    }
+    return false;
+}
+function isDependencyError(code) {
+    return code !== undefined && DEPENDENCY_ERROR_CODES.has(code);
+}
+function parsePgPosition(pos) {
+    if (pos === undefined)
+        return undefined;
+    if (typeof pos === "number" && Number.isInteger(pos) && pos > 0)
+        return pos;
+    if (typeof pos === "string") {
+        const n = Number.parseInt(pos, 10);
+        if (Number.isInteger(n) && n > 0)
+            return n;
+    }
+    return undefined;
+}
+// ---------------------------------------------------------------------------
+// Core round-based apply
+// ---------------------------------------------------------------------------
+/**
+ * Apply SQL statements to a database using iterative rounds.
+ *
+ * Algorithm:
+ * 1. Optionally set check_function_bodies = off
+ * 2. For each round, iterate over pending statements:
+ *    - On success: mark as applied
+ *    - On dependency error: defer to next round
+ *    - On environment error: skip permanently with warning
+ *    - On other error: mark as failed
+ * 3. If a round makes no progress (0 applied), stop (stuck)
+ * 4. If finalValidation is true, re-run CREATE FUNCTION/PROCEDURE
+ *    with check_function_bodies = on
+ */
+export async function roundApply(options) {
+    const { pool, statements, maxRounds = 100, disableCheckFunctionBodies = true, finalValidation = true, onRoundComplete, } = options;
+    const rounds = [];
+    const allErrors = [];
+    let totalApplied = 0;
+    let totalSkipped = 0;
+    // Track which statements still need to be applied
+    let pending = [...statements];
+    // Track statements that failed with non-dependency errors
+    const hardFailed = [];
+    // Track skipped (environment) statements
+    const skipped = [];
+    // Track applied function/procedure statements for final validation
+    const appliedFunctions = [];
+    const client = await pool.connect();
+    try {
+        // Disable function body checks to avoid false failures from
+        // functions referencing not-yet-created objects
+        if (disableCheckFunctionBodies) {
+            await client.query("SET check_function_bodies = off");
+        }
+        for (let round = 1; round <= maxRounds && pending.length > 0; round++) {
+            debugApply("round %d: %d pending", round, pending.length);
+            const roundErrors = [];
+            const deferred = [];
+            let appliedThisRound = 0;
+            let failedThisRound = 0;
+            for (const stmt of pending) {
+                try {
+                    await client.query(stmt.sql);
+                    appliedThisRound++;
+                    totalApplied++;
+                    // Track functions for final validation
+                    if (stmt.statementClass === "CREATE_FUNCTION" ||
+                        stmt.statementClass === "CREATE_PROCEDURE") {
+                        appliedFunctions.push(stmt);
+                    }
+                }
+                catch (err) {
+                    const pgErr = err;
+                    const code = pgErr.code ?? "";
+                    const message = (pgErr.message ?? "").toLowerCase();
+                    // Check if this is an environment/capability limitation
+                    if (isEnvironmentCapabilityError(code, message, stmt.statementClass)) {
+                        debugApply("skipped %s: %s", stmt.id, pgErr.message ?? code ?? "environment/capability");
+                        skipped.push(stmt);
+                        totalSkipped++;
+                        continue;
+                    }
+                    // Check if this is a dependency error (retryable)
+                    if (isDependencyError(code)) {
+                        debugApply("deferred %s: %s - %s", stmt.id, code, pgErr.message ?? "Unknown error");
+                        if (pgErr.detail)
+                            debugApply("  detail: %s", pgErr.detail);
+                        if (pgErr.hint)
+                            debugApply("  hint: %s", pgErr.hint);
+                        deferred.push(stmt);
+                        roundErrors.push({
+                            statement: stmt,
+                            code,
+                            message: pgErr.message ?? "Unknown error",
+                            isDependencyError: true,
+                            position: parsePgPosition(pgErr.position),
+                            detail: pgErr.detail,
+                            hint: pgErr.hint,
+                        });
+                        continue;
+                    }
+                    // Hard failure - non-dependency, non-environment error
+                    failedThisRound++;
+                    debugApply("failed %s: %s - %s", stmt.id, code, pgErr.message ?? "Unknown error");
+                    if (pgErr.detail)
+                        debugApply("  detail: %s", pgErr.detail);
+                    if (pgErr.hint)
+                        debugApply("  hint: %s", pgErr.hint);
+                    const stmtError = {
+                        statement: stmt,
+                        code,
+                        message: pgErr.message ?? "Unknown error",
+                        isDependencyError: false,
+                        position: parsePgPosition(pgErr.position),
+                        detail: pgErr.detail,
+                        hint: pgErr.hint,
+                    };
+                    roundErrors.push(stmtError);
+                    hardFailed.push(stmtError);
+                    allErrors.push(stmtError);
+                }
+            }
+            if (debugApply.enabled && deferred.length > 0) {
+                debugApply("Round %d complete: %d applied, %d deferred, %d failed", round, appliedThisRound, deferred.length, failedThisRound);
+                for (const e of roundErrors.filter((er) => er.isDependencyError)) {
+                    debugApply("  deferred %s: %s - %s", e.statement.id, e.code, e.message);
+                    if (e.detail)
+                        debugApply("    detail: %s", e.detail);
+                    if (e.hint)
+                        debugApply("    hint: %s", e.hint);
+                }
+            }
+            const roundResult = {
+                round,
+                applied: appliedThisRound,
+                deferred: deferred.length,
+                failed: failedThisRound,
+                errors: roundErrors,
+            };
+            rounds.push(roundResult);
+            onRoundComplete?.(roundResult);
+            // No progress this round - we're stuck
+            if (appliedThisRound === 0 && deferred.length > 0) {
+                // Collect the latest error for each stuck statement
+                const stuckStatements = deferred.map((stmt) => {
+                    const lastError = roundErrors.find((e) => e.statement.id === stmt.id);
+                    return (lastError ?? {
+                        statement: stmt,
+                        code: "UNKNOWN",
+                        message: "Deferred without a recorded error",
+                        isDependencyError: true,
+                    });
+                });
+                return {
+                    status: "stuck",
+                    totalRounds: round,
+                    totalApplied,
+                    totalSkipped,
+                    stuckStatements,
+                    errors: hardFailed.length > 0 ? hardFailed : undefined,
+                    rounds,
+                };
+            }
+            pending = deferred;
+        }
+        // If we exhausted maxRounds but still have pending, report stuck
+        if (pending.length > 0) {
+            return {
+                status: "stuck",
+                totalRounds: maxRounds,
+                totalApplied,
+                totalSkipped,
+                stuckStatements: pending.map((stmt) => ({
+                    statement: stmt,
+                    code: "MAX_ROUNDS",
+                    message: `Exceeded maximum rounds (${maxRounds})`,
+                    isDependencyError: true,
+                })),
+                errors: hardFailed.length > 0 ? hardFailed : undefined,
+                rounds,
+            };
+        }
+        // Final validation pass: re-run functions with check_function_bodies = on
+        let validationErrors;
+        if (finalValidation && appliedFunctions.length > 0) {
+            validationErrors = await validateFunctionBodies(client, appliedFunctions);
+        }
+        return {
+            status: hardFailed.length > 0
+                ? "error"
+                : validationErrors && validationErrors.length > 0
+                    ? "error"
+                    : "success",
+            totalRounds: rounds.length,
+            totalApplied,
+            totalSkipped,
+            errors: hardFailed.length > 0 ? hardFailed : undefined,
+            validationErrors: validationErrors && validationErrors.length > 0
+                ? validationErrors
+                : undefined,
+            rounds,
+        };
+    }
+    finally {
+        if (disableCheckFunctionBodies) {
+            try {
+                // Restore check_function_bodies for the connection being returned to the pool.
+                // validateFunctionBodies uses SET LOCAL inside a rolled-back transaction so it
+                // never changes the session-level value, but the SET at the start of rounds still
+                // needs to be undone here.
+                await client.query("SET check_function_bodies = on");
+            }
+            catch {
+                // Best-effort restore; connection is being released anyway
+            }
+        }
+        client.release();
+    }
+}
+/**
+ * Rewrite a CREATE FUNCTION/PROCEDURE statement to use OR REPLACE for
+ * idempotent re-execution during validation. Handles leading line-comments,
+ * block comments (pg-topo annotations), and avoids double-adding OR REPLACE.
+ */
+export function rewriteAsOrReplace(sql) {
+    return sql.replace(/^((?:(?:\s*--[^\n]*\n)|(?:\s*\/\*[\s\S]*?\*\/\s*))*\s*CREATE\s+)(?!OR\s+REPLACE\b)(FUNCTION|PROCEDURE)/i, "$1OR REPLACE $2");
+}
+/**
+ * Re-run CREATE FUNCTION/PROCEDURE statements with check_function_bodies = on
+ * using CREATE OR REPLACE to validate function bodies after all objects exist.
+ *
+ * Runs entirely inside a transaction that is always rolled back, so:
+ * - SET LOCAL search_path and check_function_bodies are transaction-scoped and
+ *   never leak to the caller's session.
+ * - The CREATE OR REPLACE changes are undone, leaving the DB exactly as it was
+ *   after the main apply rounds.
+ * - SAVEPOINTs around each statement prevent an aborted-transaction error from
+ *   blocking validation of the remaining functions.
+ */
+async function validateFunctionBodies(client, functions) {
+    const errors = [];
+    await client.query("BEGIN");
+    try {
+        // Auto-detect all user schemas so unqualified names in function bodies
+        // resolve correctly, regardless of the session's default search_path.
+        const { rows } = await client.query(`
+      SELECT string_agg(quote_ident(nspname), ', ' ORDER BY nspname) AS schemas
+      FROM pg_namespace
+      WHERE nspname NOT LIKE 'pg_%'
+        AND nspname <> 'information_schema'
+    `);
+        const detectedSchemas = rows[0]?.schemas;
+        if (detectedSchemas) {
+            debugApply("validation search_path: %s, pg_catalog", detectedSchemas);
+            await client.query(`SET LOCAL search_path = ${detectedSchemas}, pg_catalog`);
+        }
+        await client.query("SET LOCAL check_function_bodies = on");
+        for (const stmt of functions) {
+            const replaceSql = rewriteAsOrReplace(stmt.sql);
+            await client.query("SAVEPOINT validate_fn");
+            try {
+                await client.query(replaceSql);
+                await client.query("RELEASE SAVEPOINT validate_fn");
+            }
+            catch (err) {
+                await client.query("ROLLBACK TO SAVEPOINT validate_fn");
+                const pgErr = err;
+                errors.push({
+                    statement: stmt,
+                    code: pgErr.code ?? "",
+                    message: pgErr.message ?? "Unknown validation error",
+                    isDependencyError: false,
+                    position: parsePgPosition(pgErr.position),
+                    detail: pgErr.detail,
+                    hint: pgErr.hint,
+                });
+            }
+        }
+    }
+    finally {
+        // Always roll back: undoes all CREATE OR REPLACE changes and reverts the
+        // SET LOCAL search_path / check_function_bodies so nothing leaks.
+        await client.query("ROLLBACK");
+    }
+    return errors;
+}

package/dist/core/depend.d.ts

@@ -1,4 +1,4 @@
-import type { Sql } from "postgres";
+import type { Pool } from "pg";
 /**
  * Dependency type as defined in PostgreSQL's pg_depend.deptype.
  * n: normal
@@ -27,5 +27,5 @@ export interface PgDepend {
  * @param params - Object containing arrays of OIDs for filtering (user_oids, user_namespace_oids, etc.)
  * @returns Array of dependency objects with class names.
  */
-export declare function extractDepends(sql: Sql): Promise<PgDepend[]>;
+export declare function extractDepends(pool: Pool): Promise<PgDepend[]>;
 export {};

package/dist/core/depend.js

@@ -1,3 +1,4 @@
+import { sql } from "@ts-safeql/sql-tag";
 /**
  * Extract dependencies for privileges and memberships so that GRANT/REVOKE
  * operations are properly ordered with respect to their target objects/roles.
@@ -13,8 +14,8 @@
  *  - membership:<role>-><member> -> role:<role>
  *  - membership:<role>-><member> -> role:<member>
  */
-async function extractPrivilegeAndMembershipDepends(sql) {
-    const rows = await sql `
+async function extractPrivilegeAndMembershipDepends(pool) {
+    const { rows } = await pool.query(sql `
 with
   -- OBJECT PRIVILEGES (relations)
   extension_rel_oids as (
@@ -468,7 +469,7 @@ where dependent_stable_id <> referenced_stable_id
   and NOT (
     COALESCE(dep_schema, '') LIKE ANY (ARRAY['pg\\_%','information\\_schema'])
   )
-  `;
+  `);
     return rows;
 }
 /**
@@ -477,8 +478,8 @@ where dependent_stable_id <> referenced_stable_id
  * @param params - Object containing arrays of OIDs for filtering (user_oids, user_namespace_oids, etc.)
  * @returns Array of dependency objects with class names.
  */
-export async function extractDepends(sql) {
-    const dependsRows = await sql `
+export async function extractDepends(pool) {
+    const { rows: dependsRows } = await pool.query(sql `
   WITH ids AS (
     -- only the objects that actually show up in dependencies (both sides)
     SELECT DISTINCT classid, objid, objsubid FROM pg_depend WHERE deptype IN ('n','a')
@@ -1389,6 +1390,29 @@ export async function extractDepends(sql) {
     JOIN pg_namespace ns ON ns.oid = idx_rel.relnamespace
     WHERE idx_rel.relkind = 'i'
   ),
+  index_extension_deps AS (
+    -- Indexes depend on extensions that provide their operator classes
+    -- (e.g. gin_trgm_ops from pg_trgm). Without this, CREATE INDEX can be
+    -- ordered before CREATE EXTENSION when schemas sort alphabetically.
+    SELECT DISTINCT
+      format('index:%I.%I.%I', ns.nspname, tbl.relname, idx_rel.relname) AS dependent_stable_id,
+      format('extension:%I', ext.extname) AS referenced_stable_id,
+      'n'::"char" AS deptype,
+      ns.nspname AS dep_schema,
+      NULL::text AS ref_schema
+    FROM pg_class idx_rel
+    JOIN pg_index idx ON idx.indexrelid = idx_rel.oid
+    JOIN pg_class tbl ON tbl.oid = idx.indrelid
+    JOIN pg_namespace ns ON ns.oid = idx_rel.relnamespace
+    JOIN LATERAL unnest(idx.indclass) WITH ORDINALITY AS oc(oid, ord) ON true
+    JOIN pg_opclass opcl ON opcl.oid = oc.oid
+    JOIN pg_depend d ON d.classid = 'pg_opclass'::regclass
+      AND d.objid = opcl.oid
+      AND d.refclassid = 'pg_extension'::regclass
+      AND d.deptype = 'e'
+    JOIN pg_extension ext ON ext.oid = d.refobjid
+    WHERE idx_rel.relkind IN ('i', 'I')
+  ),
   ownership_deps AS (
     -- Schema ownership dependencies
     SELECT DISTINCT
@@ -1802,6 +1826,8 @@ export async function extractDepends(sql) {
     UNION ALL
     SELECT dependent_stable_id, referenced_stable_id, deptype, dep_schema, ref_schema FROM index_table_deps
     UNION ALL
+    SELECT dependent_stable_id, referenced_stable_id, deptype, dep_schema, ref_schema FROM index_extension_deps
+    UNION ALL
     SELECT dependent_stable_id, referenced_stable_id, deptype, dep_schema, ref_schema FROM ownership_deps
     UNION ALL
     SELECT dependent_stable_id, referenced_stable_id, deptype, dep_schema, ref_schema FROM publication_deps
@@ -1823,9 +1849,9 @@ export async function extractDepends(sql) {
       COALESCE(dep_schema, '') LIKE ANY (ARRAY['pg\\_%','information\\_schema'])
     )
   ORDER BY dependent_stable_id, referenced_stable_id;
-  `;
+  `);
     // Extract privilege and membership dependencies
-    const privilegeDepends = await extractPrivilegeAndMembershipDepends(sql);
+    const privilegeDepends = await extractPrivilegeAndMembershipDepends(pool);
     // Combine all dependency sources and remove duplicates
     const allDepends = new Set([...dependsRows, ...privilegeDepends]);
     return Array.from(allDepends).sort((a, b) => a.dependent_stable_id.localeCompare(b.dependent_stable_id) ||

package/dist/core/export/file-mapper.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Map changes to declarative schema file paths.
+ */
+import type { Change } from "../change.types.ts";
+import type { FilePath, Grouping, GroupingPattern } from "./types.ts";
+export declare function getFilePath(change: Change): FilePath;
+/** A compiled grouping pattern: pre-built RegExp + group name. */
+export interface CompiledPattern {
+    regex: RegExp;
+    name: string;
+}
+/** Result of compilePatterns: valid compiled patterns plus warnings for skipped/invalid regexes. */
+interface CompilePatternsResult {
+    compiled: CompiledPattern[];
+    warnings: string[];
+}
+/**
+ * Compile user-facing `GroupingPattern[]` into `CompiledPattern[]`.
+ * Strings are turned into `new RegExp(str)`. Invalid regex strings are skipped
+ * (no throw), so the returned `compiled` array may be shorter than the input.
+ * Any skipped patterns are reported in `warnings`.
+ */
+export declare function compilePatterns(patterns: GroupingPattern[]): CompilePatternsResult;
+/**
+ * Create a file mapper that applies regex-based grouping on top of the
+ * default `getFilePath` mapping.
+ *
+ * When no grouping config is provided (or it is undefined), the plain
+ * `getFilePath` function is returned unchanged.
+ */
+export declare function createFileMapper(grouping?: Grouping, onWarning?: (message: string) => void): (change: Change) => FilePath;
+/**
+ * Flatten a schema-scoped file path into one file per category.
+ *
+ * e.g. `schemas/partman/tables/template_public_events.sql`
+ *    → `schemas/partman/tables.sql`
+ *
+ * `schema.sql` is left unchanged (it is already flat).
+ */
+export declare function flattenSchema(filePath: FilePath): FilePath;
+/**
+ * Determine the group name for a change, or `null` if it should not be
+ * grouped.
+ *
+ * Resolution order:
+ *  1. Automatic partition detection -- resolve the parent table name.
+ *  2. Regex patterns -- first match wins (user controls priority by ordering).
+ *
+ * The resolved name from step 1 is fed through step 2 so that a partition
+ * parent name can itself be matched by a broader pattern (e.g. parent
+ * "kubernetes_resource_events" matches `/^kubernetes/`).
+ *
+ * If auto-detect resolved a parent but no pattern matched, the parent name
+ * is used as-is.
+ */
+export declare function resolveGroupName(change: Change, filePath: FilePath, patterns: CompiledPattern[], autoPartitions: boolean): string | null;
+/**
+ * Rewrite a `FilePath` according to the chosen grouping mode.
+ *
+ * - **single-file**: the filename becomes `{prefix}.sql` inside the original
+ *   category directory.
+ *   e.g. `schemas/public/tables/wal_verification_results_p20260107.sql`
+ *     → `schemas/public/tables/wal_verification_results.sql`
+ *
+ * - **subdirectory**: the file is moved to a prefix-named directory under the
+ *   schema root, with the category as the filename.
+ *   e.g. `schemas/public/tables/wal_verification_results_p20260107.sql`
+ *     → `schemas/public/wal_verification_results/tables.sql`
+ */
+export declare function applyGrouping(filePath: FilePath, prefix: string, mode: Grouping["mode"]): FilePath;
+export {};