dynamic_migrations 3.7.0 → 3.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 22fd29f59924e820da26858d1cf6caf04b03985ecc2fafd5f49205f5a0c2b34e
-  data.tar.gz: a8164cec22385ede8954df04e86c866bdb9d1ae4d420ba9107353f96e1a525e8
+  metadata.gz: 437a84189a939e185f89db2de8395bc1fb8dc9f159b3948a97dcc5b04c4fedb7
+  data.tar.gz: 6e1df320ac06c0b0ca9d6aeb3abf684e47b9daac4ee1def93180de801f73125b
 SHA512:
-  metadata.gz: ec4eb2339d1ce110393eae8747c9e2aabee426d58a4e3caca8e01b7dd48ddfe99291ab903c9c7ba2e474a8b509a7185850132c1934f512c8ebe710e763243ba9
-  data.tar.gz: 50076bbed1405f301476ac49f0886207f9fde401c47af062fafa4095139cd3287699b8054cac4583b97422180f8fb3be935d94ac87b1ccd071a293972811eec7
+  metadata.gz: c2ce93a8d273abfb0c2ede80b7564216a8409bcd8a43b8fe6c85043eed83d750f8a2b2e53996afdc241e13db9f6cdf2c3f52034a5e93965663684bc121237b54
+  data.tar.gz: bb3378cc475e8e3c16dd2d3a9a5754484da539d3ba1feb4839d2312c1f24cb147ea11534ba3a9bfbcda827bee4bc5ae18cc2c7a2b23bdd67c84a23503fc4fcab

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## [3.8.0](https://github.com/craigulliott/dynamic_migrations/compare/v3.7.0...v3.8.0) (2023-10-04)
+
+
+### Features
+
+* configuration option to skip removing unused extensions ([5d27e36](https://github.com/craigulliott/dynamic_migrations/commit/5d27e36363dcec552ed015a196fbcf660e6b038c))
+
+
+### Bug Fixes
+
+* adding a setting to allow customizing where the schema where materialized view structure caches are created ([f3a81f8](https://github.com/craigulliott/dynamic_migrations/commit/f3a81f850dacc40ee434de77bb1d3f3c14ea0872))
+* handling arrays of enums properly ([5d27e36](https://github.com/craigulliott/dynamic_migrations/commit/5d27e36363dcec552ed015a196fbcf660e6b038c))
+* skipping views when loading database tables ([5d27e36](https://github.com/craigulliott/dynamic_migrations/commit/5d27e36363dcec552ed015a196fbcf660e6b038c))
+
 ## [3.7.0](https://github.com/craigulliott/dynamic_migrations/compare/v3.6.16...v3.7.0) (2023-09-27)
 
 

data/lib/dynamic_migrations/postgres/server/database/differences/to_migrations/extensions.rb

@@ -18,7 +18,9 @@ module DynamicMigrations
                 # then we need to delete it
                 elsif database_extension[:exists] == true && !configuration_extension[:exists]
                   # a migration to drop the extension
-                  @generator.disable_extension extension_name
+                  if Postgres.remove_unused_extensions?
+                    @generator.disable_extension extension_name
+                  end
                 end
               end
             end

data/lib/dynamic_migrations/postgres/server/database/keys_and_unique_constraints_loader.rb

@@ -7,7 +7,7 @@ module DynamicMigrations
         module KeysAndUniqueConstraintsLoader
           def create_database_keys_and_unique_constraints_cache
             connection.exec(<<~SQL)
-              CREATE MATERIALIZED VIEW public.dynamic_migrations_keys_and_unique_constraints_cache as
+              CREATE MATERIALIZED VIEW #{Postgres.cache_schema_name}.dynamic_migrations_keys_and_unique_constraints_cache as
                 SELECT
                   c.conname AS constraint_name,
                   pg_get_constraintdef(c.oid, true) as constraint_definition,
@@ -92,16 +92,16 @@ module DynamicMigrations
               ORDER BY schema_name, table_name;
             SQL
             connection.exec(<<~SQL)
-              CREATE UNIQUE INDEX dynamic_migrations_keys_and_unique_constraints_cache_index ON public.dynamic_migrations_keys_and_unique_constraints_cache (schema_name, table_name, constraint_name);
+              CREATE UNIQUE INDEX dynamic_migrations_keys_and_unique_constraints_cache_index ON #{Postgres.cache_schema_name}.dynamic_migrations_keys_and_unique_constraints_cache (schema_name, table_name, constraint_name);
             SQL
             connection.exec(<<~SQL)
-              COMMENT ON MATERIALIZED VIEW public.dynamic_migrations_keys_and_unique_constraints_cache IS 'A cached representation of the database constraints. This is used by the dynamic migrations library and is created automatically and updated automatically after migrations have run.';
+              COMMENT ON MATERIALIZED VIEW #{Postgres.cache_schema_name}.dynamic_migrations_keys_and_unique_constraints_cache IS 'A cached representation of the database constraints. This is used by the dynamic migrations library and is created automatically and updated automatically after migrations have run.';
             SQL
           end
 
           def refresh_database_keys_and_unique_constraints_cache
             connection.exec(<<~SQL)
-              REFRESH MATERIALIZED VIEW public.dynamic_migrations_keys_and_unique_constraints_cache
+              REFRESH MATERIALIZED VIEW #{Postgres.cache_schema_name}.dynamic_migrations_keys_and_unique_constraints_cache
             SQL
           rescue PG::UndefinedTable
             create_database_keys_and_unique_constraints_cache
@@ -112,12 +112,12 @@ module DynamicMigrations
           def fetch_keys_and_unique_constraints
             begin
               rows = connection.exec(<<~SQL)
-                SELECT * FROM public.dynamic_migrations_keys_and_unique_constraints_cache
+                SELECT * FROM #{Postgres.cache_schema_name}.dynamic_migrations_keys_and_unique_constraints_cache
               SQL
             rescue PG::UndefinedTable
               create_database_keys_and_unique_constraints_cache
               rows = connection.exec(<<~SQL)
-                SELECT * FROM public.dynamic_migrations_keys_and_unique_constraints_cache
+                SELECT * FROM #{Postgres.cache_schema_name}.dynamic_migrations_keys_and_unique_constraints_cache
               SQL
             end
 

data/lib/dynamic_migrations/postgres/server/database/loaded_schemas_builder.rb

@@ -42,7 +42,9 @@ module DynamicMigrations
                 # add each table column
                 table_definition[:columns].each do |column_name, column_definition|
                   if column_definition[:is_enum]
-                    enum_schema, enum_name = column_definition[:data_type].to_s.split(".")
+                    data_type = column_definition[:data_type].to_s
+                    enum_full_name = column_definition[:is_array] ? data_type[0..-3] : data_type
+                    enum_schema, enum_name = enum_full_name.split(".")
                     enum = table.schema.database.loaded_schema(enum_schema.to_sym).enum(enum_name.to_sym)
                   else
                     enum = nil

data/lib/dynamic_migrations/postgres/server/database/schema/enums.rb

@@ -37,7 +37,9 @@ module DynamicMigrations
             # return a enum by its name, raises an error if the enum does not exist
             def enum enum_name
               raise ExpectedSymbolError, enum_name unless enum_name.is_a? Symbol
-              raise EnumDoesNotExistError unless has_enum? enum_name
+              unless has_enum? enum_name
+                raise EnumDoesNotExistError, "Enum `#{enum_name}` does not exist"
+              end
               @enums[enum_name]
             end
 

data/lib/dynamic_migrations/postgres/server/database/structure_loader.rb

@@ -7,12 +7,14 @@ module DynamicMigrations
         module StructureLoader
           def create_database_structure_cache
             connection.exec(<<~SQL)
-              CREATE MATERIALIZED VIEW public.dynamic_migrations_structure_cache AS
+              CREATE MATERIALIZED VIEW #{Postgres.cache_schema_name}.dynamic_migrations_structure_cache AS
                 SELECT
                   -- Name of the schema containing the table
                   schemata.schema_name,
                   -- Name of the table
                   tables.table_name,
+                  -- is this a real table or a view
+                  tables.table_type,
                   -- The comment which has been added to the table (if any)
                   table_description.description AS table_description,
                   -- Name of the column
@@ -24,6 +26,8 @@ module DynamicMigrations
                   -- YES if the column is possibly nullable, NO if
                   -- it is known not nullable
                   columns.is_nullable,
+                  -- Is this column an array
+                  columns.data_type = 'ARRAY' AS is_array,
                   -- The formatted data type (such as integer, char(5) or numeric(12,2)[])
                   CASE
                   WHEN tables.table_name IS NOT NULL THEN
@@ -34,12 +38,26 @@ module DynamicMigrations
                   )
                   END AS data_type,
                   -- is this an emum
-                  EXISTS (
-                    SELECT 1
-                    FROM pg_type typ
-                      INNER JOIN pg_enum enu ON typ.oid = enu.enumtypid
-                    WHERE typ.typname = columns.udt_name
-                  ) AS is_enum,
+                  CASE
+                  WHEN columns.data_type = 'ARRAY' OR columns.data_type = 'USER-DEFINED' THEN
+                  (
+                    SELECT EXISTS (
+                      SELECT 1
+                      FROM pg_type
+                      INNER JOIN pg_enum
+                        ON pg_type.oid = pg_enum.enumtypid
+                      INNER JOIN pg_namespace
+                        ON pg_namespace.oid = pg_type.typnamespace
+                      WHERE
+                        -- when the column is an array, the udt_name is the name of the enum prefixed with an underscore
+                        (columns.data_type = 'ARRAY' AND concat('_', pg_type.typname) = columns.udt_name AND pg_namespace.nspname = columns.udt_schema)
+                        -- when the column is not an array, the udt_name is the same name as the enum
+                        OR (columns.data_type = 'USER-DEFINED' AND pg_type.typname = columns.udt_name AND pg_namespace.nspname = columns.udt_schema)
+                    )
+                  )
+                  ELSE FALSE
+                  END
+                  AS is_enum,
                   -- If data_type identifies an interval type, this column contains
                   -- the specification which fields the intervals include for this
                   -- column, e.g., YEAR TO MONTH, DAY TO SECOND, etc. If no field
@@ -55,20 +73,24 @@ module DynamicMigrations
                 LEFT JOIN pg_catalog.pg_description table_description ON table_description.objoid = pg_statio_all_tables.relid AND table_description.objsubid = 0
                 -- required for the column description/comment
                 LEFT JOIN pg_catalog.pg_description column_description ON column_description.objoid = pg_statio_all_tables.relid AND column_description.objsubid = columns.ordinal_position
-                WHERE schemata.schema_name != 'information_schema'
+                WHERE
+                  -- skip internal postgres schemas
+                  schemata.schema_name != 'information_schema'
                   AND schemata.schema_name != 'postgis'
                   AND left(schemata.schema_name, 3) != 'pg_'
+                  -- only base tables (skip views), the null check is required for the left join as the schema might be empty
+                  AND (tables IS NULL OR tables.table_type = 'BASE TABLE')
                 -- order by the schema and table names alphabetically, then by the column position in the table
                 ORDER BY schemata.schema_name, tables.table_schema, columns.ordinal_position
             SQL
             connection.exec(<<~SQL)
-              COMMENT ON MATERIALIZED VIEW public.dynamic_migrations_structure_cache IS 'A cached representation of the database structure. This is used by the dynamic migrations library and is created automatically and updated automatically after migrations have run.';
+              COMMENT ON MATERIALIZED VIEW #{Postgres.cache_schema_name}.dynamic_migrations_structure_cache IS 'A cached representation of the database structure. This is used by the dynamic migrations library and is created automatically and updated automatically after migrations have run.';
             SQL
           end
 
           def refresh_database_structure_cache
             connection.exec(<<~SQL)
-              REFRESH MATERIALIZED VIEW public.dynamic_migrations_structure_cache
+              REFRESH MATERIALIZED VIEW #{Postgres.cache_schema_name}.dynamic_migrations_structure_cache
             SQL
           rescue PG::UndefinedTable
             create_database_structure_cache
@@ -79,12 +101,12 @@ module DynamicMigrations
           def fetch_structure
             begin
               rows = connection.exec(<<~SQL)
-                SELECT * FROM public.dynamic_migrations_structure_cache
+                SELECT * FROM #{Postgres.cache_schema_name}.dynamic_migrations_structure_cache
               SQL
             rescue PG::UndefinedTable
               create_database_structure_cache
               rows = connection.exec(<<~SQL)
-                SELECT * FROM public.dynamic_migrations_structure_cache
+                SELECT * FROM #{Postgres.cache_schema_name}.dynamic_migrations_structure_cache
               SQL
             end
 
@@ -109,6 +131,7 @@ module DynamicMigrations
                   column[:data_type] = row["data_type"].to_sym
                   column[:null] = row["is_nullable"] == "YES"
                   column[:is_enum] = row["is_enum"] == "t"
+                  column[:is_array] = row["is_array"] == "t"
                   column[:default] = row["column_default"]
                   column[:description] = row["column_description"]
                   column[:interval_type] = row["interval_type"].nil? ? nil : row["interval_type"].to_sym

data/lib/dynamic_migrations/postgres/server/database/validations_loader.rb

@@ -7,7 +7,7 @@ module DynamicMigrations
         module ValidationsLoader
           def create_database_validations_cache
             connection.exec(<<~SQL)
-              CREATE MATERIALIZED VIEW public.dynamic_migrations_validations_cache AS
+              CREATE MATERIALIZED VIEW #{Postgres.cache_schema_name}.dynamic_migrations_validations_cache AS
                 SELECT
                   nspname AS schema_name,
                   pg_constraint_class.relname AS table_name,
@@ -38,16 +38,16 @@ module DynamicMigrations
                     conname;
             SQL
             connection.exec(<<~SQL)
-              CREATE UNIQUE INDEX dynamic_migrations_validations_cache_index ON public.dynamic_migrations_validations_cache (schema_name, table_name, validation_name);
+              CREATE UNIQUE INDEX dynamic_migrations_validations_cache_index ON #{Postgres.cache_schema_name}.dynamic_migrations_validations_cache (schema_name, table_name, validation_name);
             SQL
             connection.exec(<<~SQL)
-              COMMENT ON MATERIALIZED VIEW public.dynamic_migrations_validations_cache IS 'A cached representation of the database validations. This is used by the dynamic migrations library and is created automatically and updated automatically after migrations have run.';
+              COMMENT ON MATERIALIZED VIEW #{Postgres.cache_schema_name}.dynamic_migrations_validations_cache IS 'A cached representation of the database validations. This is used by the dynamic migrations library and is created automatically and updated automatically after migrations have run.';
             SQL
           end
 
           def refresh_database_validations_cache
             connection.exec(<<~SQL)
-              REFRESH MATERIALIZED VIEW public.dynamic_migrations_validations_cache
+              REFRESH MATERIALIZED VIEW #{Postgres.cache_schema_name}.dynamic_migrations_validations_cache
             SQL
           rescue PG::UndefinedTable
             create_database_validations_cache
@@ -58,12 +58,12 @@ module DynamicMigrations
           def fetch_validations
             begin
               rows = connection.exec(<<~SQL)
-                SELECT * FROM public.dynamic_migrations_validations_cache
+                SELECT * FROM #{Postgres.cache_schema_name}.dynamic_migrations_validations_cache
               SQL
             rescue PG::UndefinedTable
               create_database_validations_cache
               rows = connection.exec(<<~SQL)
-                SELECT * FROM public.dynamic_migrations_validations_cache
+                SELECT * FROM #{Postgres.cache_schema_name}.dynamic_migrations_validations_cache
               SQL
             end
 

data/lib/dynamic_migrations/postgres.rb

@@ -1,8 +1,33 @@
 # frozen_string_literal: true
 
 module DynamicMigrations
-  # This module exists only to namespace Postgres functionality and
-  # make it possible to add other database/storage types in the future.
   module Postgres
+    # The default behaviour of dynamic migrations is to generate migrations
+    # which remove any unused extensions.
+    # People don't always have control over which extensions are running on the
+    # database, so this behaviour can be disabled by setting
+    # `DynamicMigrations::Postgres.remove_unused_extensions = false`
+    def self.remove_unused_extensions= value
+      @remove_unused_extensions = value
+    end
+
+    # defaults to true, but can be set to false to disable the removal of unused
+    # extensions
+    def self.remove_unused_extensions?
+      (@remove_unused_extensions.nil? || @remove_unused_extensions) ? true : false
+    end
+
+    # Dynamic Migrations creates a materialized view to store a cache representation
+    # of various parts of the database structure, by default this is created in the
+    # public schema, but this can be changed by setting the otion below.
+    def self.cache_schema_name= value
+      @cache_schema_name = value
+    end
+
+    # defaults to true, but can be set to false to disable the removal of unused
+    # extensions
+    def self.cache_schema_name
+      @cache_schema_name || :public
+    end
   end
 end

data/lib/dynamic_migrations/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DynamicMigrations
-  VERSION = "3.7.0"
+  VERSION = "3.8.0"
 end

data/lib/dynamic_migrations.rb

@@ -10,6 +10,8 @@ require "dynamic_migrations/expected_integer_error"
 require "dynamic_migrations/expected_boolean_error"
 require "dynamic_migrations/module_included_into_unexpected_target_error"
 
+require "dynamic_migrations/postgres"
+
 require "dynamic_migrations/postgres/server/database/connection"
 require "dynamic_migrations/postgres/server/database/structure_loader"
 require "dynamic_migrations/postgres/server/database/validations_loader"

data/sig/dynamic_migrations/postgres.rbs

@@ -1,4 +1,12 @@
 module DynamicMigrations
   module Postgres
+    self.@remove_unused_extensions: bool?
+    self.@cache_schema_name: Symbol?
+
+    def self.remove_unused_extensions=: (bool value) -> bool
+    def self.remove_unused_extensions?: -> bool
+
+    def self.cache_schema_name=: (Symbol value) -> Symbol
+    def self.cache_schema_name: -> Symbol
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dynamic_migrations
 version: !ruby/object:Gem::Version
-  version: 3.7.0
+  version: 3.8.0
 platform: ruby
 authors:
 - Craig Ulliott
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-09-27 00:00:00.000000000 Z
+date: 2023-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg