exwiw 0.9.3 → 0.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0dd53c3798a3084444114cbf85e6a798d656d4a8e0a0db553185673ee80ad1ca
-  data.tar.gz: 35f28a107fabb0de4efdd8a4d4701676fac5d39b6ffb0d5b0cd8dae578cd9048
+  metadata.gz: c472ab8d5098ed7cd796e8549329dadba6b4cce31b2d44b956f463699643a353
+  data.tar.gz: c5b53ba657142131d1dacdb44b3bca0117b55052384d9da2257108be6073b511
 SHA512:
-  metadata.gz: bef6a4a3ba42324f6ad06de118e1f08473c85c76dae4eb3a1573476dd9bc97d00bd2a8bfac15f3c4bdc746825eeb9c64ee047d7920d8327b0b76608ef3cb433e
-  data.tar.gz: 20ed7a044c94b0ad476a7ace6ebef261441a0e2c9e0afc0d913e4d25a540e3b8e7d3949d4b77dda0d2745bfaab986bed5ed59b1c41ecc526f4ccad48667e8369
+  metadata.gz: 79879008903ad2eff63b7c14851611fd4d6b8161726240faef77cf4533a1c172a9cbed140abe71dde1e1026de521d4d81b5d2eff074d977a8883476979f10ad0
+  data.tar.gz: bbb86bbd868783c891a1a45a337977160e569c920ded2e3a8782c098e702dabfe0f6bbbad599fd0db0b0949731a471193529c65f382de6d22c31524c776468ef

data/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## [Unreleased]
 
+## [0.9.4] - 2026-07-07
+
+### Changed
+
+- **The schema dump (`insert-000-schema.sql`) now emits DDL for every table in the source database, not just the config-scoped tables.** Previously the SQL adapters restricted the schema dump to the tables that had a config entry (`pg_dump --table <t>` / `mysqldump <db> <t...>` / a `sqlite_master` filter by `ordered_tables`), so any table without a JSON config was absent from the dump — and a restore target then failed with `relation does not exist` the moment anything referenced an unscoped table. All three SQL adapters (postgresql / mysql / sqlite) now dump the whole database's schema; a table absent from the config gets its `CREATE TABLE` (schema) but no `INSERT` (empty data), which is the intended result. Data extraction (the `INSERT` files, `validate_scope!`, the `DELETE` pass) is unchanged: `dump_schema(ordered_tables, …)` keeps its signature but uses `ordered_tables` only for logging, not to select which tables are emitted. This is a default, non-opt-in behavior change: consumer repos' baked schemas will now contain every table in the source DB.
+  - **PostgreSQL**: because a whole-database `pg_dump` emits `CREATE EXTENSION` and `CREATE TYPE … AS ENUM` itself (a `--table` dump omitted both, which is why exwiw used to prepend them by hand from `pg_extension` / `pg_enum`), the manual prepend and its two helper queries are removed. Their robustness is preserved by two new `DdlPostprocessor` passes applied in place: enum types are wrapped in a `DO $exwiw$ … EXCEPTION WHEN duplicate_object THEN NULL … $exwiw$` block (so a re-restore does not fail), and each `CREATE EXTENSION` is wrapped in a `DO $$ … EXCEPTION WHEN feature_not_supported OR invalid_schema_name THEN RAISE WARNING … $$` block (so a target that cannot provide the extension warns-and-skips rather than aborting; `insufficient_privilege` is still not caught). A managed-platform or `pglogical` extension the source happens to have is now handled by this graceful skip instead of being filtered out by name.
+  - `strip_triggers` is still applied unchanged. A whole-database dump now also contains the `CREATE FUNCTION` definitions that triggers reference, so exwiw could in principle keep triggers instead of stripping them — but this release does not change that behavior.
+
 ## [0.9.3] - 2026-07-02
 
 ### Fixed

data/lib/exwiw/adapter/mysql_adapter.rb

@@ -80,11 +80,12 @@ module Exwiw
       end
 
       def dump_schema(ordered_tables, output_path)
-        table_names = ordered_tables.map(&:name)
-        if table_names.empty?
-          File.write(output_path, "-- Auto-generated by exwiw. No tables in scope.\n")
-          return
-        end
+        # Full-database schema dump: emit DDL for every table in the database,
+        # not just the config-scoped tables. A table absent from the config gets
+        # its schema here but no data (empty INSERT), which is the intended
+        # result — the restore target then has every relation even when exwiw was
+        # configured to extract only a subset. `ordered_tables` no longer selects
+        # which tables are emitted; it is kept only for the log line.
 
         # The mysqldump binary is invoked directly (not via the mysql2/trilogy
         # driver), so point EXWIW_MYSQLDUMP at a specific binary when the one on
@@ -111,11 +112,10 @@ module Exwiw
           *gtid_flags,
           '--compact',
           @connection_config.database_name,
-          *table_names,
         ]
         env = { 'MYSQL_PWD' => @connection_config.password.to_s }
 
-        @logger.debug("  Running #{mysqldump_bin} for #{table_names.size} table(s)...")
+        @logger.debug("  Running #{mysqldump_bin} for the whole database (#{@connection_config.database_name})...")
         stdout, stderr, status =
           begin
             Open3.capture3(env, *cmd)
@@ -140,7 +140,7 @@ module Exwiw
           file.puts(idempotent)
           file.puts("SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS;")
         end
-        @logger.info("  Wrote schema for #{table_names.size} table(s) to #{output_path}.")
+        @logger.info("  Wrote full-database schema to #{output_path} (#{ordered_tables.size} table(s) in scope for data).")
       end
 
       def pre_insert_sql(_table)

data/lib/exwiw/adapter/postgresql_adapter.rb

@@ -100,12 +100,14 @@ module Exwiw
       def dump_schema(ordered_tables, output_path)
         require 'open3'
 
-        table_names = ordered_tables.map(&:name)
-        if table_names.empty?
-          File.write(output_path, "-- Auto-generated by exwiw. No tables in scope.\n")
-          return
-        end
-
+        # Full-database schema dump: emit DDL for every table in the database,
+        # not just the config-scoped tables. A table absent from the config
+        # gets its schema here but no data (empty INSERT), which is the intended
+        # result — the restore target then has every relation even when exwiw
+        # was configured to extract only a subset, so a later reference to an
+        # unscoped table does not fail with "relation does not exist".
+        # `ordered_tables` no longer selects which tables are emitted; it is kept
+        # only for the log line (how many tables are in scope for data).
         cmd = [
           'pg_dump',
           "--host=#{@connection_config.host}",
@@ -114,12 +116,11 @@ module Exwiw
           '--schema-only',
           '--no-owner',
           '--no-acl',
-          *table_names.flat_map { |t| ['--table', t] },
           @connection_config.database_name,
         ]
         env = { 'PGPASSWORD' => @connection_config.password.to_s }
 
-        @logger.debug("  Running pg_dump for #{table_names.size} table(s)...")
+        @logger.debug("  Running pg_dump for the whole database (#{@connection_config.database_name})...")
         stdout, stderr, status = Open3.capture3(env, *cmd)
         unless status.success?
           if stderr.include?('command not found') || stderr.empty?
@@ -128,37 +129,17 @@ module Exwiw
           raise "pg_dump failed (exit #{status.exitstatus}): #{stderr}"
         end
 
-        # Enums are prepended first, then extensions — so extensions end up at the top of the output.
-        enum_types = query_enum_types(table_names)
-        unless enum_types.empty?
-          enum_ddl = DdlPostprocessor.create_type_enum_statements(enum_types)
-          @logger.debug("  Found #{enum_types.size} enum type(s) to prepend.")
-          stdout = enum_ddl + stdout
-        end
-
-        extensions = query_extensions
-        unless extensions.empty?
-          ext_ddl = extensions.map do |extname, schema|
-            stmt = "CREATE EXTENSION IF NOT EXISTS #{connection.quote_ident(extname)}"
-            stmt += " SCHEMA #{connection.quote_ident(schema)}" unless schema == "public"
-            # Best-effort prepend: a restore target that genuinely cannot create the
-            # extension should not abort the whole restore. Two such cases are caught:
-            #   feature_not_supported (0A000) -- the extension's binaries are unavailable
-            #   invalid_schema_name   (3F000) -- the extension's required schema is absent
-            # insufficient_privilege (42501) is deliberately NOT caught: a restore role
-            # lacking CREATE privilege is a misconfiguration to fix, not to skip silently.
-            # The skip is re-raised as a WARNING so it surfaces in the restore logs
-            # instead of vanishing.
-            warning = connection.escape_literal("exwiw: skipped CREATE EXTENSION #{extname} (SQLSTATE %): %")
-            "DO $$ BEGIN #{stmt}; " \
-              "EXCEPTION WHEN feature_not_supported OR invalid_schema_name THEN " \
-              "RAISE WARNING #{warning}, SQLSTATE, SQLERRM; END $$;"
-          end.join("\n") + "\n\n"
-          @logger.debug("  Found #{extensions.size} extension(s) to prepend.")
-          stdout = ext_ddl + stdout
-        end
-
+        # A full-database dump emits CREATE EXTENSION and CREATE TYPE ... AS ENUM
+        # itself (a `--table` dump omitted both, which is why they used to be
+        # prepended by hand), but pg_dump's bare forms are neither idempotent nor
+        # restore-tolerant. Wrap them in place to restore the previous
+        # robustness: enums swallow duplicate_object on re-restore; extensions
+        # warn-and-skip feature_not_supported / invalid_schema_name so a target
+        # that cannot provide the extension (e.g. pglogical's schema absent, or a
+        # managed-platform extension) does not abort the restore.
         idempotent = stdout
+        idempotent = DdlPostprocessor.wrap_create_type_enum_in_do_block(idempotent)
+        idempotent = DdlPostprocessor.wrap_create_extension_in_do_block(idempotent)
         idempotent = DdlPostprocessor.add_if_not_exists_to_create_schema(idempotent)
         idempotent = DdlPostprocessor.add_if_not_exists_to_create_sequence(idempotent)
         idempotent = DdlPostprocessor.add_if_not_exists_to_create_table(idempotent)
@@ -170,7 +151,7 @@ module Exwiw
           file.puts("-- Auto-generated by exwiw via pg_dump. Idempotent DDL for postgresql.")
           file.write(idempotent)
         end
-        @logger.info("  Wrote schema for #{table_names.size} table(s) to #{output_path}.")
+        @logger.info("  Wrote full-database schema to #{output_path} (#{ordered_tables.size} table(s) in scope for data).")
       end
 
       # The INSERT header for this adapter. PostgreSQL uses bare identifiers.
@@ -533,62 +514,6 @@ module Exwiw
         end
       end
 
-      private def query_extensions
-        # Skip plpgsql (always present) and managed-platform bookkeeping extensions
-        # (google_*/rds_*/aiven_*). pglogical is also skipped: it is a logical-
-        # replication mechanism of the source, not part of the data being copied,
-        # and its dedicated `pglogical` schema is typically absent on the restore
-        # target — so prepending CREATE EXTENSION for it only breaks the restore.
-        sql = <<~SQL
-          SELECT e.extname, n.nspname
-          FROM pg_extension e
-          JOIN pg_namespace n ON n.oid = e.extnamespace
-          WHERE e.extname != 'plpgsql'
-            AND e.extname != 'pglogical'
-            AND e.extname NOT LIKE 'google\\_%' ESCAPE '\\'
-            AND e.extname NOT LIKE 'rds\\_%' ESCAPE '\\'
-            AND e.extname NOT LIKE 'aiven\\_%' ESCAPE '\\'
-          ORDER BY e.extname
-        SQL
-        connection.exec(sql).map { |row| [row["extname"], row["nspname"]] }
-      end
-
-      private def query_enum_types(table_names)
-        return [] if table_names.empty?
-
-        placeholders = table_names.each_with_index.map { |_, i| "$#{i + 1}" }.join(', ')
-        sql = <<~SQL
-          SELECT
-            n.nspname  AS type_schema,
-            t.typname  AS type_name,
-            array_agg(e.enumlabel ORDER BY e.enumsortorder) AS enum_labels
-          FROM pg_type t
-          JOIN pg_namespace n ON n.oid = t.typnamespace
-          JOIN pg_enum e      ON e.enumtypid = t.oid
-          WHERE t.typtype = 'e'
-            AND t.oid IN (
-              SELECT a.atttypid
-              FROM pg_attribute a
-              JOIN pg_class c ON c.oid = a.attrelid
-              WHERE c.relname IN (#{placeholders})
-                AND a.attnum > 0
-                AND NOT a.attisdropped
-            )
-          GROUP BY n.nspname, t.typname
-          ORDER BY n.nspname, t.typname
-        SQL
-
-        result = connection.exec_params(sql, table_names)
-        decoder = PG::TextDecoder::Array.new
-        result.map do |row|
-          {
-            schema: row['type_schema'],
-            name: row['type_name'],
-            labels: decoder.decode(row['enum_labels']),
-          }
-        end
-      end
-
       private def column_pg_type(table_name, column_name)
         @column_type_cache ||= {}
         cache_key = [table_name, column_name]

data/lib/exwiw/adapter/sqlite_adapter.rb

@@ -81,25 +81,27 @@ module Exwiw
 
       def dump_schema(ordered_tables, output_path)
         @logger.debug("  Reading schema from sqlite_master...")
-        target_names = ordered_tables.map(&:name)
-        # `sqlite_master` row order preserves table creation order, which is also
-        # the dependency order produced by ActiveRecord-style migrations. To respect
-        # the caller-provided order, we partition tables / their owned indexes by
-        # ordered_tables.
+        # Full-database schema dump: emit DDL for every table in the database,
+        # not just the config-scoped tables. A table absent from the config gets
+        # its schema here but no data (empty INSERT), which is the intended
+        # result — the restore target then has every relation even when exwiw was
+        # configured to extract only a subset. `ordered_tables` no longer selects
+        # which tables are emitted; it is kept only for the log line.
+        #
+        # `sqlite_master` row order preserves creation order, which is also the
+        # dependency order produced by ActiveRecord-style migrations (and lists a
+        # table before its owned indexes/triggers), so emitting rows in that
+        # order keeps each table's DDL ahead of its dependents.
         all = connection.execute(<<~SQL)
           SELECT type, name, tbl_name, sql FROM sqlite_master
           WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%'
         SQL
 
-        tables_by_name   = all.select { |type, _, _, _| type == 'table'   }.to_h { |_, name, _, sql| [name, sql] }
         indexes_by_owner = all.select { |type, _, _, _| type == 'index'   }.group_by { |_, _, tbl, _| tbl }
         triggers_by_owner = all.select { |type, _, _, _| type == 'trigger' }.group_by { |_, _, tbl, _| tbl }
 
         statements = []
-        target_names.each do |name|
-          table_sql = tables_by_name[name]
-          next unless table_sql
-
+        all.select { |type, _, _, _| type == 'table' }.each do |_, name, _, table_sql|
           statements << finalize_stmt(DdlPostprocessor.add_if_not_exists_to_create_table(table_sql.strip))
           (indexes_by_owner[name] || []).each do |_, _, _, idx_sql|
             statements << finalize_stmt(DdlPostprocessor.add_if_not_exists_to_create_index(idx_sql.strip))
@@ -113,7 +115,7 @@ module Exwiw
           file.puts("-- Auto-generated by exwiw. Idempotent CREATE statements for SQLite.")
           file.puts(statements.join("\n"))
         end
-        @logger.info("  Wrote #{statements.size} schema statement(s) to #{output_path}.")
+        @logger.info("  Wrote #{statements.size} schema statement(s) to #{output_path} (#{ordered_tables.size} table(s) in scope for data).")
       end
 
       private def finalize_stmt(stmt)

data/lib/exwiw/ddl_postprocessor.rb

@@ -64,6 +64,48 @@ module Exwiw
       sql.gsub(/^[ \t]*CREATE\s+(?:OR\s+REPLACE\s+)?(?:CONSTRAINT\s+)?TRIGGER\b[^;]*;\r?\n?/i, "")
     end
 
+    # A bare `CREATE TYPE ... AS ENUM (...)` (as a full-database pg_dump emits,
+    # unlike a `--table` dump, which omits enum types) is not idempotent: a
+    # second restore raises `duplicate_object`. Wrap each in a DO block that
+    # swallows that error, matching the form of #create_type_enum_statements.
+    # Enum labels never contain a semicolon or an unescaped `)`, so the match
+    # ends at the first `);` after `AS ENUM (`.
+    CREATE_TYPE_ENUM_RE = /^[ \t]*CREATE\s+TYPE\b.+?\bAS\s+ENUM\s*\(.+?\)\s*;/mi.freeze
+
+    def wrap_create_type_enum_in_do_block(sql)
+      sql.gsub(CREATE_TYPE_ENUM_RE) do |stmt|
+        <<~SQL.chomp
+          DO $exwiw$ BEGIN
+            #{stmt.strip}
+          EXCEPTION WHEN duplicate_object THEN NULL;
+          END $exwiw$;
+        SQL
+      end
+    end
+
+    # A bare `CREATE EXTENSION ...;` (as a full-database pg_dump emits, unlike a
+    # `--table` dump, which omits extensions) has no graceful skip: a restore
+    # target that cannot create the extension aborts the whole restore. Wrap
+    # each in a DO block that catches only the two "cannot provide it here"
+    # cases — feature_not_supported (0A000, binaries absent) and
+    # invalid_schema_name (3F000, required schema absent) — and re-raises them
+    # as a WARNING so the skip surfaces in the restore logs. insufficient_
+    # privilege (42501) is deliberately NOT caught: a restore role lacking
+    # CREATE privilege is a misconfiguration to fix, not to skip silently.
+    CREATE_EXTENSION_RE = /^[ \t]*CREATE\s+EXTENSION\b(?:\s+IF\s+NOT\s+EXISTS)?\s+(?<name>"[^"]+"|[^\s;]+)[^;]*;/i.freeze
+
+    def wrap_create_extension_in_do_block(sql)
+      sql.gsub(CREATE_EXTENSION_RE) do
+        stmt = Regexp.last_match(0).strip
+        extname = Regexp.last_match(:name).delete('"')
+        warning = "exwiw: skipped CREATE EXTENSION #{extname} (SQLSTATE %): %"
+        warning_literal = "'#{warning.gsub("'", "''")}'"
+        "DO $$ BEGIN #{stmt} " \
+          "EXCEPTION WHEN feature_not_supported OR invalid_schema_name THEN " \
+          "RAISE WARNING #{warning_literal}, SQLSTATE, SQLERRM; END $$;"
+      end
+    end
+
     # Generate idempotent CREATE TYPE ... AS ENUM statements.
     # +enum_types+ is an Array of Hashes with keys :schema, :name, :labels.
     def create_type_enum_statements(enum_types)

data/lib/exwiw/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Exwiw
-  VERSION = "0.9.3"
+  VERSION = "0.9.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: exwiw
 version: !ruby/object:Gem::Version
-  version: 0.9.3
+  version: 0.9.4
 platform: ruby
 authors:
 - Shia