dynamic_migrations 1.0.0

data/lib/dynamic_migrations/postgres/server/database/differences.rb

@@ -0,0 +1,292 @@
+# frozen_string_literal: true
+
+module DynamicMigrations
+  module Postgres
+    class Server
+      class Database
+        class Differences
+          class ExpectedDatabaseError < StandardError
+          end
+
+          class TableRequiredError < StandardError
+          end
+
+          class SchemaRequiredError < StandardError
+          end
+
+          def initialize database
+            raise ExpectedDatabaseError, database unless database.is_a? Database
+            @database = database
+          end
+
+          # return a hash representing any differenced betweek the loaded and configured
+          # versions of the current database
+          def to_h
+            {
+              configuration: self.class.compare_schemas(@database.configured_schemas_hash, @database.loaded_schemas_hash),
+              database: self.class.compare_schemas(@database.loaded_schemas_hash, @database.configured_schemas_hash)
+            }
+          end
+
+          def self.compare_schemas schemas, comparison_schemas
+            result = {}
+            # the base schemas
+            schemas.each do |schema_name, schema|
+              # compare this schema to the equivilent in the comparison list
+              # note that the comparison may be nil
+              result[schema_name] = compare_schema schema, comparison_schemas[schema_name]
+            end
+            # look for any in the comparison list which were not in the base list
+            comparison_schemas.each do |schema_name, schema|
+              unless result.key? schema_name
+                result[schema_name] = {
+                  exists: false
+                }
+              end
+            end
+            result
+          end
+
+          # compare two schemas and return an object which represents the provided `schema` and
+          # any differences between it and the provided `comparison_schema`
+          def self.compare_schema schema, comparison_schema
+            raise SchemaRequiredError if schema.nil?
+
+            comparison_tables = comparison_schema.nil? ? {} : comparison_schema.tables_hash
+            {
+              exists: true,
+              tables: compare_tables(schema.tables_hash, comparison_tables)
+            }
+          end
+
+          # compare two hash representations of a set of tables and return
+          # an object which represents the provided `tables` and any differences
+          # between it and the `comparison_tables`
+          def self.compare_tables tables, comparison_tables
+            result = {}
+            # the base tables
+            tables.each do |table_name, table|
+              # compare this table to the equivilent in the comparison list
+              # note that the comparison may be nil
+              result[table_name] = compare_table(table, comparison_tables[table_name])
+            end
+            # look for any in the comparison list which were not in the base list
+            comparison_tables.each do |table_name, table|
+              unless result.key? table_name
+                result[table_name] = {
+                  exists: false
+                }
+              end
+            end
+            result
+          end
+
+          # compare two tables and return an object which represents the provided `table` and
+          # any differences between it and the provided `comparison_table`
+          def self.compare_table table, comparison_table
+            raise TableRequiredError if table.nil?
+
+            primary_key = table.has_primary_key? ? table.primary_key : nil
+            if comparison_table
+              comparison_primary_key = comparison_table.has_primary_key? ? comparison_table.primary_key : nil
+              comparison_columns = comparison_table.columns_hash
+              comparison_validations = comparison_table.validations_hash
+              comparison_foreign_key_constraints = comparison_table.foreign_key_constraints_hash
+              comparison_unique_constraints = comparison_table.unique_constraints_hash
+            else
+              comparison_primary_key = {}
+              comparison_columns = {}
+              comparison_validations = {}
+              comparison_foreign_key_constraints = {}
+              comparison_unique_constraints = {}
+            end
+            {
+              exists: true,
+              description: {
+                value: table.description,
+                matches: (comparison_table && comparison_table.description == table.description) || false
+              },
+              primary_key: compare_record(primary_key, comparison_primary_key, [
+                :primary_key_name,
+                :index_type
+              ]),
+              columns: compare_columns(table.columns_hash, comparison_columns),
+              validations: compare_validations(table.validations_hash, comparison_validations),
+              foreign_key_constraints: compare_foreign_key_constraints(table.foreign_key_constraints_hash, comparison_foreign_key_constraints),
+              unique_constraints: compare_unique_constraints(table.unique_constraints_hash, comparison_unique_constraints)
+            }
+          end
+
+          # compare two hash representations of a set of columns and return
+          # an object which represents the provided `columns` and any differences
+          # between it and the `comparison_columns`
+          def self.compare_columns columns, comparison_columns
+            result = {}
+            # the base columns
+            columns.each do |column_name, column|
+              # compare this column to the equivilent in the comparison list
+              result[column_name] = compare_record column, comparison_columns[column_name], [
+                :data_type,
+                :null,
+                :default,
+                :description,
+                :character_maximum_length,
+                :character_octet_length,
+                :numeric_precision,
+                :numeric_precision_radix,
+                :numeric_scale,
+                :datetime_precision,
+                :interval_type,
+                :udt_schema,
+                :udt_name,
+                :updatable
+              ]
+            end
+            # look for any columns in the comparison list which were not in the base list
+            comparison_columns.each do |column_name, column|
+              unless result.key? column_name
+                result[column_name] = {
+                  exists: false
+                }
+              end
+            end
+            result
+          end
+
+          # compare two hash representations of a set of unique_constraints and return
+          # an object which represents the provided `unique_constraints` and any differences
+          # between it and the `comparison_unique_constraints`
+          def self.compare_unique_constraints unique_constraints, comparison_unique_constraints
+            result = {}
+            # the base unique_constraints
+            unique_constraints.each do |unique_constraint_name, unique_constraint|
+              # compare this unique_constraint to the equivilent in the comparison list
+              result[unique_constraint_name] = compare_record unique_constraint, comparison_unique_constraints[unique_constraint_name], [
+                :column_names,
+                :index_type,
+                :deferrable,
+                :initially_deferred
+              ]
+            end
+            # look for any unique_constraints in the comparison list which were not in the base list
+            comparison_unique_constraints.each do |unique_constraint_name, unique_constraint|
+              unless result.key? unique_constraint_name
+                result[unique_constraint_name] = {
+                  exists: false
+                }
+              end
+            end
+            result
+          end
+
+          # compare two hash representations of a set of indexes and return
+          # an object which represents the provided `indexes` and any differences
+          # between it and the `comparison_indexes`
+          def self.compare_indexes indexes, comparison_indexes
+            result = {}
+            # the base indexes
+            indexes.each do |index_name, index|
+              # compare this index to the equivilent in the comparison list
+              result[index_name] = compare_record index, comparison_indexes[index_name], [
+                :column_names,
+                :unique,
+                :where,
+                :type,
+                :deferrable,
+                :initially_deferred,
+                :order,
+                :nulls_position
+              ]
+            end
+            # look for any indexes in the comparison list which were not in the base list
+            comparison_indexes.each do |index_name, index|
+              unless result.key? index_name
+                result[index_name] = {
+                  exists: false
+                }
+              end
+            end
+            result
+          end
+
+          # compare two hash representations of a set of validations and return
+          # an object which represents the provided `validations` and any differences
+          # between it and the `comparison_validations`
+          def self.compare_validations validations, comparison_validations
+            result = {}
+            # the base validations
+            validations.each do |validation_name, validation|
+              # compare this validation to the equivilent in the comparison list
+              result[validation_name] = compare_record validation, comparison_validations[validation_name], [
+                :check_clause,
+                :column_names,
+                :deferrable,
+                :initially_deferred
+              ]
+            end
+            # look for any validations in the comparison list which were not in the base list
+            comparison_validations.each do |validation_name, validation|
+              unless result.key? validation_name
+                result[validation_name] = {
+                  exists: false
+                }
+              end
+            end
+            result
+          end
+
+          # compare two hash representations of a set of foreign key constraints and return
+          # an object which represents the provided `foreign_key_constraints` and any differences
+          # between it and the `comparison_foreign_key_constraints`
+          def self.compare_foreign_key_constraints foreign_key_constraints, comparison_foreign_key_constraints
+            result = {}
+            # the base foreign_key_constraints
+            foreign_key_constraints.each do |foreign_key_constraint_name, foreign_key_constraint|
+              # compare this foreign_key_constraint to the equivilent in the comparison list
+              result[foreign_key_constraint_name] = compare_record foreign_key_constraint, comparison_foreign_key_constraints[foreign_key_constraint_name], [
+                :column_names,
+                :foreign_schema_name,
+                :foreign_table_name,
+                :foreign_column_names,
+                :deferrable,
+                :initially_deferred
+              ]
+            end
+            # look for any foreign_key_constraints in the comparison list which were not in the base list
+            comparison_foreign_key_constraints.each do |foreign_key_constraint_name, foreign_key_constraint|
+              unless result.key? foreign_key_constraint_name
+                result[foreign_key_constraint_name] = {
+                  exists: false
+                }
+              end
+            end
+            result
+          end
+
+          # Accepts an optional base and comparison objects, and a list of methods.
+          # Returns a hash representing the base record and all the data which it
+          # is returned for each mehod in the provided method list and any differences
+          # it and the comparison.
+          def self.compare_record base, comparison, method_list
+            if base.nil?
+              {
+                exists: false
+              }
+            else
+              result = {}
+              method_list.each do |method_name|
+                matches = (comparison && comparison.send(method_name) == base.send(method_name)) || false
+                result[method_name] = {
+                  value: base.send(method_name),
+                  matches: matches
+                }
+              end
+              result[:exists] = true
+              result
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dynamic_migrations/postgres/server/database/keys_and_unique_constraints_loader.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module DynamicMigrations
+  module Postgres
+    class Server
+      class Database
+        module KeysAndUniqueConstraintsLoader
+          def create_database_keys_and_unique_constraints_cache
+            connection.exec(<<~SQL)
+              CREATE MATERIALIZED VIEW public.dynamic_migrations_keys_and_unique_constraints_cache as
+                SELECT
+                  c.conname AS constraint_name,
+                  CASE c.contype
+                    WHEN 'f'::"char" THEN 'FOREIGN_KEY'::text
+                    WHEN 'p'::"char" THEN 'PRIMARY_KEY'::text
+                    WHEN 'u'::"char" THEN 'UNIQUE'::text
+                  END::information_schema.character_data AS constraint_type,
+                  sch.nspname AS schema_name,
+                  tbl.relname AS table_name,
+                  ARRAY_AGG(col.attname ORDER BY u.attposition) AS column_names,
+                  f_sch.nspname AS foreign_schema_name,
+                  f_tbl.relname AS foreign_table_name,
+                  -- null if is required to prevent indexes and unique constraints from being included
+                  NULLIF(ARRAY_AGG(f_col.attname ORDER BY f_u.attposition), ARRAY[null]::name[]) AS foreign_column_names,
+                  c.condeferrable as deferrable,
+                  c.condeferred as initially_deferred,
+                  am.amname as index_type,
+                  1 as table_version
+                FROM pg_constraint c
+                LEFT JOIN LATERAL UNNEST(c.conkey)
+                  WITH ORDINALITY AS u(attnum, attposition)
+                  ON TRUE
+                LEFT JOIN LATERAL UNNEST(c.confkey)
+                  WITH ORDINALITY AS f_u(attnum, attposition)
+                  ON f_u.attposition = u.attposition
+                JOIN pg_class tbl
+                  ON
+                    tbl.oid = c.conrelid
+                    AND left(tbl.relname, 3) != 'pg_'
+                JOIN pg_namespace sch
+                  ON
+                    sch.oid = tbl.relnamespace
+                    AND sch.nspname != 'information_schema'
+                    AND sch.nspname != 'postgis'
+                    AND left(sch.nspname, 3) != 'pg_'
+                LEFT JOIN pg_attribute col
+                  ON
+                    (col.attrelid = tbl.oid
+                    AND col.attnum = u.attnum)
+                LEFT JOIN pg_class f_tbl
+                  ON
+                    f_tbl.oid = c.confrelid
+                    AND left(f_tbl.relname, 3) != 'pg_'
+                LEFT JOIN pg_namespace f_sch
+                  ON
+                    f_sch.oid = f_tbl.relnamespace
+                    AND f_sch.nspname != 'information_schema'
+                    AND f_sch.nspname != 'postgis'
+                    AND left(f_sch.nspname, 3) != 'pg_'
+                LEFT JOIN pg_attribute f_col
+                  ON
+                    f_col.attrelid = f_tbl.oid
+                    AND f_col.attnum = f_u.attnum
+
+                -- joins below to get the index type
+                LEFT JOIN pg_class index_cls ON index_cls.relname = c.conname AND index_cls.relnamespace = sch.oid
+                LEFT JOIN pg_index on index_cls.oid = pg_index.indexrelid AND tbl.oid = pg_index.indrelid
+                LEFT JOIN pg_am am ON am.oid=index_cls.relam
+
+                WHERE
+                -- only FOREIGN_KEY, UNIQUE or PRIMARY_KEY
+                c.contype in ('f', 'u', 'p')
+
+              GROUP BY constraint_name, constraint_type, condeferrable, condeferred, schema_name, table_name, foreign_schema_name, foreign_table_name, am.amname
+              ORDER BY schema_name, table_name;
+            SQL
+            connection.exec(<<~SQL)
+              CREATE UNIQUE INDEX dynamic_migrations_keys_and_unique_constraints_cache_index ON public.dynamic_migrations_keys_and_unique_constraints_cache (schema_name, table_name, constraint_name);
+            SQL
+            connection.exec(<<~SQL)
+              COMMENT ON MATERIALIZED VIEW public.dynamic_migrations_keys_and_unique_constraints_cache IS 'A cached representation of the database constraints. This is used by the dynamic migrations library and is created automatically and updated automatically after migrations have run.';
+            SQL
+          end
+
+          # fetch all required data from the database and build and return a
+          # useful hash representing the keys and indexes of your database
+          def fetch_keys_and_unique_constraints
+            begin
+              rows = connection.exec_params(<<~SQL)
+                SELECT * FROM public.dynamic_migrations_keys_and_unique_constraints_cache
+              SQL
+            rescue PG::UndefinedTable
+              create_database_keys_and_unique_constraints_cache
+              rows = connection.exec_params(<<~SQL)
+                SELECT * FROM public.dynamic_migrations_keys_and_unique_constraints_cache
+              SQL
+            end
+
+            schemas = {}
+            rows.each do |row|
+              schema_name = row["schema_name"].to_sym
+              schema = schemas[schema_name] ||= {}
+
+              table_name = row["table_name"].to_sym
+              table = schema[table_name] ||= {}
+
+              constraint_type = row["constraint_type"].to_sym
+              constraints = table[constraint_type] ||= {}
+
+              constraint_name = row["constraint_name"].to_sym
+
+              column_names = row["column_names"].gsub(/\A\{/, "").gsub(/\}\Z/, "").split(",").map { |column_name| column_name.to_sym }
+
+              if constraint_type == :FOREIGN_KEY
+                foreign_schema_name = row["foreign_schema_name"].to_sym
+                foreign_table_name = row["foreign_table_name"].to_sym
+                foreign_column_names = row["foreign_column_names"].gsub(/\A\{/, "").gsub(/\}\Z/, "").split(",").map { |column_name| column_name.to_sym }
+              else
+                foreign_schema_name = nil
+                foreign_table_name = nil
+                foreign_column_names = nil
+              end
+
+              deferrable = row["deferrable"] == "TRUE"
+              initially_deferred = row["initially_deferred"] == "TRUE"
+
+              index_type = if row["index_type"].nil?
+                nil
+              else
+                row["index_type"].to_sym
+              end
+
+              constraints[constraint_name] = {
+                column_names: column_names,
+                foreign_schema_name: foreign_schema_name,
+                foreign_table_name: foreign_table_name,
+                foreign_column_names: foreign_column_names,
+                deferrable: deferrable,
+                initially_deferred: initially_deferred,
+                index_type: index_type
+              }
+            end
+            schemas
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dynamic_migrations/postgres/server/database/loaded_schemas.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module DynamicMigrations
+  module Postgres
+    class Server
+      class Database
+        module LoadedSchemas
+          class LoadedSchemaAlreadyExistsError < StandardError
+          end
+
+          class LoadedSchemaDoesNotExistError < StandardError
+          end
+
+          # adds a new loaded schema for this database
+          def add_loaded_schema schema_name
+            raise ExpectedSymbolError, schema_name unless schema_name.is_a? Symbol
+            if has_loaded_schema? schema_name
+              raise(LoadedSchemaAlreadyExistsError, "Loaded schema #{schema_name} already exists")
+            end
+            included_target = self
+            if included_target.is_a? Database
+              @loaded_schemas[schema_name] = Schema.new :database, included_target, schema_name
+            else
+              raise ModuleIncludedIntoUnexpectedTargetError, included_target
+            end
+          end
+
+          # returns the loaded schema object for the provided schema name, and raises an
+          # error if the schema does not exist
+          def loaded_schema schema_name
+            raise ExpectedSymbolError, schema_name unless schema_name.is_a? Symbol
+            raise LoadedSchemaDoesNotExistError unless has_loaded_schema? schema_name
+            @loaded_schemas[schema_name]
+          end
+
+          # returns true if this table has a loaded schema with the provided name, otherwise false
+          def has_loaded_schema? schema_name
+            raise ExpectedSymbolError, schema_name unless schema_name.is_a? Symbol
+            @loaded_schemas.key? schema_name
+          end
+
+          # returns an array of this tables loaded schemas
+          def loaded_schemas
+            @loaded_schemas.values
+          end
+
+          # returns a hash of this tables loaded schemas, keyed by schema name
+          def loaded_schemas_hash
+            @loaded_schemas
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dynamic_migrations/postgres/server/database/loaded_schemas_builder.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module DynamicMigrations
+  module Postgres
+    class Server
+      class Database
+        module LoadedSchemasBuilder
+          class UnexpectedConstrintTypeError < StandardError
+          end
+
+          # recursively process the database and build all the schemas,
+          # tables and columns
+          def recursively_build_schemas_from_database
+            validations = fetch_validations
+            fetch_structure.each do |schema_name, schema_definition|
+              schema = add_loaded_schema schema_name
+              schema_validations = validations[schema_name]
+
+              schema_definition[:tables].each do |table_name, table_definition|
+                table = schema.add_table table_name, table_definition[:description]
+                table_validations = schema_validations && schema_validations[table_name]
+
+                # add each table column
+                table_definition[:columns].each do |column_name, column_definition|
+                  # we only need these for arrays and user-defined types
+                  # (user-defined is usually ENUMS)
+                  if [:ARRAY, :"USER-DEFINED"].include? column_definition[:data_type]
+                    udt_schema = column_definition[:udt_schema]
+                    udt_name = column_definition[:udt_name]
+                  else
+                    udt_schema = nil
+                    udt_name = nil
+                  end
+
+                  table.add_column column_name, column_definition[:data_type],
+                    null: column_definition[:null],
+                    default: column_definition[:default],
+                    description: column_definition[:description],
+                    character_maximum_length: column_definition[:character_maximum_length],
+                    character_octet_length: column_definition[:character_octet_length],
+                    numeric_precision: column_definition[:numeric_precision],
+                    numeric_precision_radix: column_definition[:numeric_precision_radix],
+                    numeric_scale: column_definition[:numeric_scale],
+                    datetime_precision: column_definition[:datetime_precision],
+                    udt_schema: udt_schema,
+                    udt_name: udt_name
+                end
+
+                # add any validations
+                table_validations&.each do |validation_name, validation_definition|
+                  table.add_validation validation_name, validation_definition[:columns], validation_definition[:check_clause]
+                end
+              end
+            end
+
+            # now that the structure has been loaded, we can add keys (foreign
+            # keys need to be added last, because they can depend on tables from
+            # different schemas)
+            fetch_keys_and_unique_constraints.each do |schema_name, schema_definition|
+              schema_definition.each do |table_name, keys_and_unique_constraints|
+                table = loaded_schema(schema_name).table(table_name)
+                keys_and_unique_constraints.each do |constraint_type, constraint_definitions|
+                  constraint_definitions.each do |constraint_name, constraint_definition|
+                    case constraint_type
+                    when :PRIMARY_KEY
+                      table.add_primary_key constraint_name, constraint_definition[:column_names], index_type: constraint_definition[:index_type]
+
+                    when :FOREIGN_KEY
+                      table.add_foreign_key_constraint constraint_name, constraint_definition[:column_names], constraint_definition[:foreign_schema_name], constraint_definition[:foreign_table_name], constraint_definition[:foreign_column_names], deferrable: constraint_definition[:deferrable], initially_deferred: constraint_definition[:initially_deferred]
+
+                    when :UNIQUE
+                      table.add_unique_constraint constraint_name, constraint_definition[:column_names], deferrable: constraint_definition[:deferrable], initially_deferred: constraint_definition[:initially_deferred], index_type: constraint_definition[:index_type]
+
+                    else
+                      raise UnexpectedConstrintTypeError, constraint_type
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dynamic_migrations/postgres/server/database/schema/table/column.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module DynamicMigrations
+  module Postgres
+    class Server
+      class Database
+        class Schema
+          class Table
+            # This class represents a single column within a postgres table
+            class Column < Source
+              class ExpectedTableError < StandardError
+              end
+
+              attr_reader :table
+              attr_reader :column_name
+              attr_reader :description
+              attr_reader :null
+              attr_reader :default
+              attr_reader :data_type
+              attr_reader :character_maximum_length
+              attr_reader :character_octet_length
+              attr_reader :numeric_precision
+              attr_reader :numeric_precision_radix
+              attr_reader :numeric_scale
+              attr_reader :datetime_precision
+              attr_reader :interval_type
+              attr_reader :udt_schema
+              attr_reader :udt_name
+              attr_reader :updatable
+
+              # initialize a new object to represent a column in a postgres table
+              def initialize source, table, column_name, data_type, null: true, default: nil, description: nil, character_maximum_length: nil, character_octet_length: nil, numeric_precision: nil, numeric_precision_radix: nil, numeric_scale: nil, datetime_precision: nil, interval_type: nil, udt_schema: nil, udt_name: nil, updatable: true
+                super source
+                raise ExpectedTableError, table unless table.is_a? Table
+                @table = table
+
+                raise ExpectedSymbolError, column_name unless column_name.is_a? Symbol
+                @column_name = column_name
+
+                @data_type = data_type
+
+                @null = null
+
+                @default = default
+
+                unless description.nil?
+                  raise ExpectedStringError, description unless description.is_a? String
+                  @description = description
+                end
+
+                DataTypes.validate_column_properties!(data_type,
+                  character_maximum_length: character_maximum_length,
+                  character_octet_length: character_octet_length,
+                  numeric_precision: numeric_precision,
+                  numeric_precision_radix: numeric_precision_radix,
+                  numeric_scale: numeric_scale,
+                  datetime_precision: datetime_precision,
+                  interval_type: interval_type,
+                  udt_schema: udt_schema,
+                  udt_name: udt_name)
+
+                @character_maximum_length = character_maximum_length
+                @character_octet_length = character_octet_length
+                @numeric_precision = numeric_precision
+                @numeric_precision_radix = numeric_precision_radix
+                @numeric_scale = numeric_scale
+                @datetime_precision = datetime_precision
+                @interval_type = interval_type
+                @udt_schema = udt_schema
+                @udt_name = udt_name
+                @updatable = updatable
+              end
+
+              # return true if this column has a description, otherwise false
+              def has_description?
+                !@description.nil?
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end