@supabase/pg-delta 1.0.0-alpha.3 → 1.0.0-alpha.5

package/src/cli/commands/declarative-apply.ts

@@ -0,0 +1,380 @@
+/**
+ * Declarative-apply command - apply a declarative SQL schema to a database
+ * using pg-topo static analysis + round-based execution.
+ */
+
+import { readFile } from "node:fs/promises";
+import { buildCommand, type CommandContext } from "@stricli/core";
+import chalk from "chalk";
+import { loadDeclarativeSchema } from "../../core/declarative-apply/discover-sql.ts";
+import {
+  applyDeclarativeSchema,
+  type DeclarativeApplyResult,
+  type RoundResult,
+} from "../../core/declarative-apply/index.ts";
+import {
+  buildDiagnosticDisplayItems,
+  type DiagnosticDisplayEntry,
+  formatStatementError,
+  positionToLineColumn,
+  requiredObjectKeyFromDiagnostic,
+  resolveSqlFilePath,
+} from "../utils/apply-display.ts";
+
+export const declarativeApplyCommand = buildCommand({
+  parameters: {
+    flags: {
+      path: {
+        kind: "parsed",
+        brief:
+          "Path to the declarative schema directory (containing .sql files) or a single .sql file",
+        parse: String,
+      },
+      target: {
+        kind: "parsed",
+        brief: "Target database connection URL to apply the schema to",
+        parse: String,
+      },
+      "max-rounds": {
+        kind: "parsed",
+        brief:
+          "Maximum number of application rounds before giving up (default: 100)",
+        parse: Number,
+        optional: true,
+      },
+      "no-validate-functions": {
+        kind: "boolean",
+        brief: "Skip final function body validation pass",
+        optional: true,
+      },
+      verbose: {
+        kind: "boolean",
+        brief: "Show detailed per-round progress",
+        optional: true,
+      },
+      "ungroup-diagnostics": {
+        kind: "boolean",
+        brief:
+          "Show full per-diagnostic detail instead of grouped summary output",
+        optional: true,
+      },
+    },
+    aliases: {
+      p: "path",
+      t: "target",
+      v: "verbose",
+    },
+  },
+  docs: {
+    brief: "Apply a declarative SQL schema to a database",
+    fullDescription: `
+Apply SQL files from a declarative schema directory to a target database.
+
+Uses pg-topo for static dependency analysis and topological ordering,
+then applies statements round-by-round to handle any remaining
+dependency gaps. Statements that fail with dependency errors are
+deferred to subsequent rounds until all succeed or no progress is made.
+
+Function body checks are disabled during rounds to avoid false failures
+from functions referencing not-yet-created objects. A final validation
+pass re-runs all function/procedure definitions with body checks enabled.
+
+Exit codes:
+  0 - Success (all statements applied)
+  1 - Error (hard failures or validation errors)
+  2 - Stuck (dependency cycle or unresolvable ordering)
+
+Tip: Use DEBUG=pg-delta:declarative-apply for detailed defer/skip/fail logs (which statements are deferred and why).
+    `.trim(),
+  },
+  async func(
+    this: CommandContext,
+    flags: {
+      path: string;
+      target: string;
+      "max-rounds"?: number;
+      "no-validate-functions"?: boolean;
+      verbose?: boolean;
+      "ungroup-diagnostics"?: boolean;
+    },
+  ) {
+    const verbose = !!flags.verbose;
+    const ungroupDiagnostics = !!flags["ungroup-diagnostics"];
+
+    const onRoundComplete = verbose
+      ? (round: RoundResult) => {
+          const parts = [
+            `Round ${round.round}:`,
+            chalk.green(`${round.applied} applied`),
+          ];
+          if (round.deferred > 0) {
+            parts.push(chalk.yellow(`${round.deferred} deferred`));
+          }
+          if (round.failed > 0) {
+            parts.push(chalk.red(`${round.failed} failed`));
+          }
+          this.process.stdout.write(`${parts.join("  ")}\n`);
+        }
+      : undefined;
+
+    this.process.stdout.write(`Analyzing SQL files in ${flags.path}...\n`);
+
+    let content: Array<{ filePath: string; sql: string }>;
+    try {
+      content = await loadDeclarativeSchema(flags.path);
+    } catch (error) {
+      this.process.stderr.write(
+        `Error: ${error instanceof Error ? error.message : String(error)}\n`,
+      );
+      process.exitCode = 1;
+      return;
+    }
+
+    if (content.length === 0) {
+      this.process.stderr.write(
+        `No .sql files found in '${flags.path}'. Pass a directory containing .sql files or a single .sql file.\n`,
+      );
+      process.exitCode = 1;
+      return;
+    }
+
+    let result: DeclarativeApplyResult;
+    try {
+      result = await applyDeclarativeSchema({
+        content,
+        targetUrl: flags.target,
+        maxRounds: flags["max-rounds"],
+        validateFunctionBodies: !flags["no-validate-functions"],
+        onRoundComplete,
+      });
+    } catch (error) {
+      this.process.stderr.write(
+        `Error: ${error instanceof Error ? error.message : String(error)}\n`,
+      );
+      process.exitCode = 1;
+      return;
+    }
+
+    // Report pg-topo diagnostics grouped by severity (least noisy first).
+    // UNKNOWN_STATEMENT_CLASS is always hidden; DUPLICATE_PRODUCER,
+    // CYCLE_EDGE_SKIPPED, and UNRESOLVED_DEPENDENCY are verbose-only.
+    const diagnosticDisplayOrder: Record<string, number> = {
+      UNKNOWN_STATEMENT_CLASS: 0,
+      DUPLICATE_PRODUCER: 1,
+      CYCLE_EDGE_SKIPPED: 2,
+      UNRESOLVED_DEPENDENCY: 3,
+    };
+    const diagnosticColor: Record<string, (s: string) => string> = {
+      DUPLICATE_PRODUCER: chalk.yellow,
+      CYCLE_EDGE_SKIPPED: chalk.red,
+      UNRESOLVED_DEPENDENCY: chalk.dim,
+    };
+    const verboseOnlyCodes = new Set([
+      "UNRESOLVED_DEPENDENCY",
+      "DUPLICATE_PRODUCER",
+      "CYCLE_EDGE_SKIPPED",
+    ]);
+    const warnings = result.diagnostics
+      .filter(
+        (d) =>
+          d.code !== "UNKNOWN_STATEMENT_CLASS" &&
+          (verbose || !verboseOnlyCodes.has(d.code)),
+      )
+      .sort(
+        (a, b) =>
+          (diagnosticDisplayOrder[a.code] ?? 99) -
+          (diagnosticDisplayOrder[b.code] ?? 99),
+      );
+    if (warnings.length > 0 && verbose) {
+      const fileContentCache = new Map<string, string>();
+      for (const diag of warnings) {
+        const id = diag.statementId;
+        if (
+          id &&
+          id.sourceOffset != null &&
+          id.filePath &&
+          !fileContentCache.has(id.filePath)
+        ) {
+          // Try to resolve the exact file path of the statement to get the exact location of the error
+          try {
+            const fullPath = await resolveSqlFilePath(flags.path, id.filePath);
+            const content = await readFile(fullPath, "utf-8");
+            fileContentCache.set(id.filePath, content);
+          } catch {
+            // Fall back to statementIndex display
+          }
+        }
+      }
+
+      this.process.stderr.write(
+        chalk.yellow(
+          `\n${warnings.length} diagnostic(s) from static analysis:\n`,
+        ),
+      );
+      const entries: DiagnosticDisplayEntry[] = warnings.map((diag) => {
+        let location: string | undefined;
+        if (diag.statementId) {
+          const id = diag.statementId;
+          const offset = id.sourceOffset;
+          const content =
+            offset != null ? fileContentCache.get(id.filePath) : undefined;
+          if (content != null && offset != null) {
+            const { line, column } = positionToLineColumn(content, offset + 1);
+            location = `${id.filePath}:${line}:${column}`;
+          } else {
+            location = `${id.filePath}:${id.statementIndex}`;
+          }
+        }
+        return {
+          diagnostic: diag,
+          location,
+          requiredObjectKey: requiredObjectKeyFromDiagnostic(diag),
+        };
+      });
+      const displayItems = buildDiagnosticDisplayItems(
+        entries,
+        !ungroupDiagnostics,
+      );
+
+      let lastCode = "";
+      const previewLimit = 5;
+      for (const item of displayItems) {
+        if (item.code !== lastCode) {
+          if (lastCode !== "") {
+            this.process.stderr.write("\n");
+          }
+          lastCode = item.code;
+        }
+        const colorFn = diagnosticColor[item.code] ?? chalk.yellow;
+        const location =
+          item.locations.length > 0 ? ` (${item.locations[0]})` : "";
+        const occurrences =
+          !ungroupDiagnostics && item.locations.length > 1
+            ? ` x${item.locations.length}`
+            : "";
+        this.process.stderr.write(
+          colorFn(
+            `  [${item.code}]${location}${occurrences} ${item.message}\n`,
+          ),
+        );
+        if (!ungroupDiagnostics && item.requiredObjectKey) {
+          this.process.stderr.write(
+            colorFn(`    -> Object: ${item.requiredObjectKey}\n`),
+          );
+        }
+        if (!ungroupDiagnostics && item.locations.length > 1) {
+          for (const locationEntry of item.locations.slice(0, previewLimit)) {
+            this.process.stderr.write(colorFn(`    at ${locationEntry}\n`));
+          }
+          const remaining = item.locations.length - previewLimit;
+          if (remaining > 0) {
+            this.process.stderr.write(
+              colorFn(`    ... and ${remaining} more location(s)\n`),
+            );
+          }
+        }
+        if (item.suggestedFix) {
+          this.process.stderr.write(
+            colorFn(`    -> Fix: ${item.suggestedFix}\n`),
+          );
+        }
+      }
+      this.process.stderr.write("\n");
+    }
+
+    const { apply } = result;
+
+    // Summary
+    this.process.stdout.write("\n");
+    this.process.stdout.write(
+      `Statements: ${result.totalStatements} total, ${apply.totalApplied} applied`,
+    );
+    if (apply.totalSkipped > 0) {
+      this.process.stdout.write(`, ${apply.totalSkipped} skipped`);
+    }
+    this.process.stdout.write("\n");
+    this.process.stdout.write(`Rounds: ${apply.totalRounds}\n`);
+
+    switch (apply.status) {
+      case "success": {
+        this.process.stdout.write(
+          chalk.green("All statements applied successfully.\n"),
+        );
+        if (apply.validationErrors && apply.validationErrors.length > 0) {
+          this.process.stderr.write(
+            chalk.yellow(
+              `\n${apply.validationErrors.length} function body validation error(s):\n`,
+            ),
+          );
+          for (const err of apply.validationErrors) {
+            const formatted = await formatStatementError(err, flags.path);
+            this.process.stderr.write(chalk.yellow(formatted));
+            this.process.stderr.write("\n\n");
+          }
+          process.exitCode = 1;
+        } else {
+          process.exitCode = 0;
+        }
+        break;
+      }
+
+      case "stuck": {
+        this.process.stderr.write(
+          chalk.red(
+            `\nStuck after ${apply.totalRounds} round(s). ${apply.stuckStatements?.length ?? 0} statement(s) could not be applied:\n`,
+          ),
+        );
+        if (apply.stuckStatements) {
+          for (const stuck of apply.stuckStatements) {
+            const formatted = await formatStatementError(stuck, flags.path);
+            this.process.stderr.write(chalk.red(formatted));
+            this.process.stderr.write("\n\n");
+          }
+        }
+        if (apply.errors && apply.errors.length > 0) {
+          this.process.stderr.write(
+            chalk.red(
+              `\nAdditionally, ${apply.errors.length} statement(s) had non-dependency errors:\n`,
+            ),
+          );
+          for (const err of apply.errors) {
+            const formatted = await formatStatementError(err, flags.path);
+            this.process.stderr.write(chalk.red(formatted));
+            this.process.stderr.write("\n\n");
+          }
+        }
+        process.exitCode = 2;
+        break;
+      }
+
+      case "error": {
+        this.process.stderr.write(
+          chalk.red(
+            `\nCompleted with errors. ${apply.errors?.length ?? 0} statement(s) failed:\n`,
+          ),
+        );
+        if (apply.errors) {
+          for (const err of apply.errors) {
+            const formatted = await formatStatementError(err, flags.path);
+            this.process.stderr.write(chalk.red(formatted));
+            this.process.stderr.write("\n\n");
+          }
+        }
+        if (apply.validationErrors && apply.validationErrors.length > 0) {
+          this.process.stderr.write(
+            chalk.yellow(
+              `\n${apply.validationErrors.length} function body validation error(s):\n`,
+            ),
+          );
+          for (const err of apply.validationErrors) {
+            const formatted = await formatStatementError(err, flags.path);
+            this.process.stderr.write(chalk.yellow(formatted));
+            this.process.stderr.write("\n\n");
+          }
+        }
+        process.exitCode = 1;
+        break;
+      }
+    }
+  },
+});

package/src/cli/commands/declarative-export.ts

@@ -0,0 +1,330 @@
+/**
+ * Declarative export command - export a declarative SQL schema from a database diff.
+ */
+
+import { mkdir, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { buildCommand, type CommandContext } from "@stricli/core";
+import chalk from "chalk";
+import type { CatalogSnapshot } from "../../core/catalog.snapshot.ts";
+import { exportDeclarativeSchema } from "../../core/export/index.ts";
+import type { Grouping, GroupingPattern } from "../../core/export/types.ts";
+import type { FilterDSL } from "../../core/integrations/filter/dsl.ts";
+import type { ChangeFilter } from "../../core/integrations/filter/filter.types.ts";
+import type { SerializeDSL } from "../../core/integrations/serialize/dsl.ts";
+import type { ChangeSerializer } from "../../core/integrations/serialize/serialize.types.ts";
+import { createPlan } from "../../core/plan/index.ts";
+import type { SqlFormatOptions } from "../../core/plan/sql-format.ts";
+import {
+  assertSafePath,
+  buildFileTree,
+  computeFileDiff,
+  formatExportSummary,
+} from "../utils/export-display.ts";
+import { loadIntegrationDSL } from "../utils/integrations.ts";
+import { isPostgresUrl, loadCatalogFromFile } from "../utils/resolve-input.ts";
+
+function parseJsonFlag<T>(label: string, value: string): T {
+  try {
+    return JSON.parse(value) as T;
+  } catch (error) {
+    throw new Error(
+      `Invalid ${label} JSON: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+}
+
+export const declarativeExportCommand = buildCommand({
+  parameters: {
+    flags: {
+      source: {
+        kind: "parsed",
+        brief:
+          "Source (current state): postgres URL or catalog snapshot file path. Omit to export all objects from target.",
+        parse: String,
+        optional: true,
+      },
+      target: {
+        kind: "parsed",
+        brief:
+          "Target (desired state): postgres URL or catalog snapshot file path",
+        parse: String,
+      },
+      output: {
+        kind: "parsed",
+        brief: "Output directory path for declarative schema files",
+        parse: String,
+      },
+      integration: {
+        kind: "parsed",
+        brief:
+          "Integration name (e.g., 'supabase') or path to integration JSON file",
+        parse: String,
+        optional: true,
+      },
+      filter: {
+        kind: "parsed",
+        brief: 'Filter DSL as inline JSON (e.g., \'{"schema":"public"}\')',
+        parse: (v: string) => parseJsonFlag<FilterDSL>("filter", v),
+        optional: true,
+      },
+      serialize: {
+        kind: "parsed",
+        brief:
+          'Serialize DSL as inline JSON array (e.g., \'[{"when":{"type":"schema"},"options":{"skipAuthorization":true}}]\')',
+        parse: (v: string) => parseJsonFlag<SerializeDSL>("serialize", v),
+        optional: true,
+      },
+      "grouping-mode": {
+        kind: "enum",
+        brief: "How grouped entities are organized on disk",
+        values: ["single-file", "subdirectory"] as const,
+        optional: true,
+      },
+      "group-patterns": {
+        kind: "parsed",
+        brief:
+          'JSON array of {pattern, name} objects (e.g., \'[{"pattern":"^auth","name":"auth"}]\')',
+        parse: (v: string) => {
+          const parsed = parseJsonFlag<GroupingPattern[]>("group-patterns", v);
+          if (!Array.isArray(parsed)) {
+            throw new Error("group-patterns must be a JSON array");
+          }
+          return parsed;
+        },
+        optional: true,
+      },
+      "flat-schemas": {
+        kind: "parsed",
+        brief:
+          "Comma-separated list of schemas to flatten (e.g., partman,pgboss,audit)",
+        parse: String,
+        optional: true,
+      },
+      "format-options": {
+        kind: "parsed",
+        brief:
+          'SQL format options as inline JSON (e.g., \'{"keywordCase":"lower","maxWidth":180}\')',
+        parse: (v: string) =>
+          parseJsonFlag<SqlFormatOptions>("format-options", v),
+        optional: true,
+      },
+      force: {
+        kind: "boolean",
+        brief: "Remove entire output directory before writing",
+        optional: true,
+      },
+      "dry-run": {
+        kind: "boolean",
+        brief: "Show tree and summary without writing files",
+        optional: true,
+      },
+      "diff-focus": {
+        kind: "boolean",
+        brief:
+          "Show only files that changed (created/updated/deleted) in the tree",
+        optional: true,
+      },
+      verbose: {
+        kind: "boolean",
+        brief: "Show detailed output",
+        optional: true,
+      },
+    },
+    aliases: {
+      s: "source",
+      t: "target",
+      o: "output",
+    },
+  },
+  docs: {
+    brief: "Export a declarative schema from a database diff",
+    fullDescription: `
+Export a declarative SQL schema by comparing two databases (source → target).
+Writes .sql files to the output directory, grouped by object type and optional
+grouping rules.
+
+When --source is omitted, all objects from the target database are exported
+(equivalent to diffing from an empty database).
+
+Flags:
+  source         - Source database connection URL (optional; omit for full export)
+  target         - Target database connection URL (desired state)
+  output         - Directory path for generated .sql files
+  integration    - Integration name or path (e.g., supabase) for filter/serialize
+  filter         - Filter DSL as JSON to include/exclude changes
+  serialize      - Serialize DSL as JSON array for custom SQL generation
+  grouping-mode  - single-file or subdirectory for grouped entities
+  group-patterns - JSON array of {pattern, name} for name-based grouping
+  flat-schemas   - Comma-separated schemas to merge into one file per category
+  format-options - SQL format options as JSON
+  force          - Remove output directory before writing (full replace)
+  dry-run        - Show tree and summary only, do not write files
+  diff-focus     - Show only changed files (created/updated/deleted) in the tree
+  verbose        - Show detailed output
+
+After export, a tip is printed with the command to apply the schema to an empty database.
+    `.trim(),
+  },
+  async func(
+    this: CommandContext,
+    flags: {
+      source?: string;
+      target: string;
+      output: string;
+      integration?: string;
+      filter?: FilterDSL;
+      serialize?: SerializeDSL;
+      "grouping-mode"?: "single-file" | "subdirectory";
+      "group-patterns"?: GroupingPattern[];
+      "flat-schemas"?: string;
+      "format-options"?: SqlFormatOptions;
+      force?: boolean;
+      "dry-run"?: boolean;
+      "diff-focus"?: boolean;
+      verbose?: boolean;
+    },
+  ) {
+    const { compileSerializeDSL } = await import(
+      "../../core/integrations/serialize/dsl.ts"
+    );
+
+    let filterOption: FilterDSL | ChangeFilter | undefined = flags.filter;
+    let serializeOption: SerializeDSL | ChangeSerializer | undefined =
+      flags.serialize;
+    let integrationEmptyCatalog: CatalogSnapshot | undefined;
+    if (flags.integration) {
+      const integrationDSL = await loadIntegrationDSL(flags.integration);
+      filterOption = filterOption ?? integrationDSL.filter;
+      serializeOption = serializeOption ?? integrationDSL.serialize;
+      integrationEmptyCatalog = integrationDSL.emptyCatalog;
+    }
+
+    const resolvedSource = flags.source
+      ? isPostgresUrl(flags.source)
+        ? flags.source
+        : await loadCatalogFromFile(flags.source)
+      : integrationEmptyCatalog
+        ? (await import("../../core/catalog.snapshot.ts")).deserializeCatalog(
+            integrationEmptyCatalog,
+          )
+        : null;
+
+    const resolvedTarget = isPostgresUrl(flags.target)
+      ? flags.target
+      : await loadCatalogFromFile(flags.target);
+
+    // Pass raw DSL to createPlan (not pre-compiled functions).
+    // createPlan compiles them internally and uses the DSL type to correctly
+    // determine cascade behavior: DSL filters disable cascading by default
+    // (unless cascade:true is set), preventing unintended exclusion of
+    // changes that depend on filtered objects (e.g. RLS policies that
+    // reference auth.uid() when the auth schema is filtered out).
+    const planResult = await createPlan(resolvedSource, resolvedTarget, {
+      filter: filterOption,
+      serialize: serializeOption,
+      skipDefaultPrivilegeSubtraction: true,
+    });
+
+    if (!planResult) {
+      this.process.stdout.write("No changes detected.\n");
+      return;
+    }
+
+    const hasGrouping =
+      flags["grouping-mode"] !== undefined ||
+      (flags["group-patterns"] !== undefined &&
+        flags["group-patterns"].length > 0) ||
+      (flags["flat-schemas"] !== undefined && flags["flat-schemas"].length > 0);
+
+    let grouping: Grouping | undefined;
+    if (hasGrouping) {
+      grouping = {
+        mode: flags["grouping-mode"] ?? "single-file",
+        groupPatterns: flags["group-patterns"],
+        autoGroupPartitions: true,
+        flatSchemas:
+          flags["flat-schemas"] !== undefined
+            ? flags["flat-schemas"]
+                .split(",")
+                .map((s) => s.trim())
+                .filter(Boolean)
+            : undefined,
+      };
+    }
+
+    const serializeFn =
+      serializeOption !== undefined
+        ? compileSerializeDSL(serializeOption)
+        : undefined;
+
+    const output = exportDeclarativeSchema(planResult, {
+      integration:
+        serializeFn !== undefined ? { serialize: serializeFn } : undefined,
+      formatOptions: flags["format-options"] ?? undefined,
+      grouping,
+      onWarning: (msg) => {
+        this.process.stderr.write(chalk.yellow(`Warning: ${msg}\n`));
+      },
+    });
+
+    const outputDir = path.resolve(flags.output);
+    const applyTip = (dir: string) =>
+      `\nTip: To apply this schema to an empty database, run:\n  pgdelta declarative apply --path ${dir} --target <database_url>\n`;
+    const diff = await computeFileDiff(outputDir, output.files);
+
+    this.process.stdout.write("\n");
+    this.process.stdout.write(
+      `${buildFileTree(
+        output.files.map((f) => f.path),
+        path.basename(outputDir) || outputDir,
+        { diff, diffFocus: !!flags["diff-focus"] },
+      )}\n`,
+    );
+    this.process.stdout.write("\n");
+    this.process.stdout.write(
+      `${chalk.green("+")} created   ${chalk.yellow("~")} updated   ${chalk.red("-")} deleted\n`,
+    );
+    this.process.stdout.write("\n");
+
+    const summary = formatExportSummary(diff, !!flags["dry-run"]);
+    if (summary) {
+      this.process.stdout.write(`${summary}\n`);
+    }
+
+    const totalChanges = planResult.sortedChanges.length;
+    const totalStatements = output.files.reduce((s, f) => s + f.statements, 0);
+    this.process.stdout.write(
+      `Changes: ${totalChanges} | Files: ${output.files.length} | Statements: ${totalStatements}\n`,
+    );
+
+    if (flags["dry-run"]) {
+      this.process.stdout.write(chalk.dim("\n(dry-run: no files written)\n"));
+      this.process.stdout.write(chalk.cyan(applyTip(outputDir)));
+      return;
+    }
+
+    if (flags.force) {
+      await rm(outputDir, { recursive: true, force: true });
+      await mkdir(outputDir, { recursive: true });
+    } else if (diff.deleted.length > 0) {
+      this.process.stderr.write(
+        chalk.yellow(
+          `Warning: ${diff.deleted.length} existing file(s) will no longer be present. Use --force to replace the output directory.\n`,
+        ),
+      );
+    }
+
+    for (const file of output.files) {
+      assertSafePath(file.path, outputDir);
+      const filePath = path.join(outputDir, file.path);
+      await mkdir(path.dirname(filePath), { recursive: true });
+      await writeFile(filePath, file.sql);
+    }
+
+    this.process.stdout.write(
+      chalk.green(`Wrote ${output.files.length} file(s) to ${outputDir}\n`),
+    );
+    this.process.stdout.write(chalk.cyan(applyTip(outputDir)));
+  },
+});