schema_ferry 0.2.0 → 0.3.0

data/lib/schema_ferry/core/table_converter.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+module SchemaFerry
+  module Core
+    class TableConverter
+      include Support::Warnings
+      include IdentifierShortenable
+
+      UNSUPPORTED_INDEX_TYPES = %i[fulltext spatial].freeze
+      SPATIAL_SQL_TYPES = /\A(?:geometry|point|linestring|polygon|multipoint|multilinestring|multipolygon|
+                              geometrycollection)\z/ix
+
+      def initialize(raw, rule:, foreign_keys:, fk_columns:, column_converter:, enum_check:)
+        @raw              = raw
+        @name             = raw[:name]
+        @rule             = rule
+        @foreign_keys     = foreign_keys
+        @fk_columns       = fk_columns
+        @column_converter = column_converter
+        @enum_check       = enum_check
+      end
+
+      def convert
+        check_table_name_length!
+
+        TableSchema.new(
+          name:              @name,
+          primary_key:       @raw[:primary_key],
+          pk_type:           convert_pk_type,
+          pk_limit:          convert_pk_limit,
+          comment:           @raw[:comment].presence,
+          columns:           convert_columns,
+          indexes:           convert_indexes,
+          foreign_keys:      convert_foreign_keys,
+          check_constraints: build_check_constraints
+        )
+      end
+
+      private
+
+      def check_table_name_length!
+        return if @name.bytesize <= MAX_BYTES
+
+        raise ConversionError, "table name #{@name.inspect} exceeds PostgreSQL's #{MAX_BYTES}-byte " \
+                               "identifier limit. Exclude it with ignore_table :#{@name}, or rename it in MySQL."
+      end
+
+      def pk_column
+        pk = @raw[:primary_key]
+        pk.is_a?(String) ? @raw[:columns].find { |c| c[:name] == pk } : nil
+      end
+
+      # A single-column PK is rendered via create_table's id: option, so it is
+      # excluded from the column list. Composite PK columns must stay:
+      # primary_key: [...] does not create them.
+      def data_columns
+        @raw[:columns].reject { |c| c[:name] == @raw[:primary_key] }
+      end
+
+      # MySQL BIGINT comes through AR as :integer with limit 8, so the sql_type
+      # is the only reliable source for the id: option.
+      def convert_pk_type
+        col = pk_column
+        return nil if col.nil?
+        return col[:type] unless col[:type] == :integer
+
+        sql_type = col[:sql_type].to_s
+        if sql_type.include?("unsigned")
+          if sql_type.start_with?("bigint")
+            emit_warning "table #{@name}: BIGINT UNSIGNED primary key has no PostgreSQL " \
+                         "equivalent; using signed bigint (values above 2^63-1 will not fit)."
+          end
+          :bigint
+        elsif sql_type.start_with?("bigint")
+          :bigint
+        else
+          :integer
+        end
+      end
+
+      def convert_pk_limit
+        col = pk_column
+        col[:limit] if col && col[:type] == :string
+      end
+
+      def convert_columns
+        data_columns
+          .reject { |c| @rule.ignored_columns.include?(c[:name]) }
+          .map do |c|
+            check_spatial_type!(c)
+            @column_converter.call(c, @name, @rule, @fk_columns)
+          end
+      end
+
+      def check_spatial_type!(col)
+        return if @rule.column_type_overrides.key?(col[:name]) # handled downstream, like any other override
+
+        if col[:sql_type].to_s.match?(SPATIAL_SQL_TYPES)
+          raise ConversionError, "#{@name}.#{col[:name]}: MySQL #{col[:sql_type]} columns have no " \
+                                 "PostgreSQL equivalent without PostGIS, which schema_ferry does not " \
+                                 "manage. Exclude it with ignore_column :#{col[:name]}."
+        end
+      end
+
+      def convert_indexes
+        @raw[:indexes]
+          .reject { |idx| @rule.ignored_indexes.include?(idx[:name]) }
+          .reject { |idx| idx[:columns].intersect?(@rule.ignored_columns) }
+          .map do |idx|
+            check_unsupported_index_type!(idx)
+            build_index_schema(idx)
+          end
+      end
+
+      def check_unsupported_index_type!(idx)
+        if UNSUPPORTED_INDEX_TYPES.include?(idx[:type])
+          raise ConversionError, "#{@name}: #{idx[:type].to_s.upcase} index #{idx[:name].inspect} " \
+                                 "has no PostgreSQL equivalent. Exclude it with ignore_index :#{idx[:name]}."
+        end
+      end
+
+      def build_index_schema(idx)
+        IndexSchema.new(
+          name:    shorten_identifier(idx[:name], kind: "index", table: @name),
+          columns: idx[:columns],
+          unique:  idx[:unique],
+          using:   idx[:using],
+          # AR reports "no prefix lengths / orders" as an empty hash; rendering
+          # a literal lengths: {} would make ridgepole re-create the index on
+          # every run.
+          lengths: idx[:lengths].presence,
+          orders:  idx[:orders].presence
+        )
+      end
+
+      def convert_foreign_keys
+        @foreign_keys.map { |fk| build_fk_schema(fk) }
+      end
+
+      def build_fk_schema(foreign_key)
+        ForeignKeySchema.new(
+          from_table:  foreign_key[:from_table],
+          to_table:    foreign_key[:to_table],
+          column:      foreign_key[:column],
+          primary_key: foreign_key[:primary_key],
+          on_update:   foreign_key[:on_update],
+          on_delete:   foreign_key[:on_delete],
+          name:        shorten_identifier(foreign_key[:name], kind: "foreign key", table: foreign_key[:from_table])
+        )
+      end
+
+      def build_check_constraints
+        return [] unless @enum_check
+
+        @enum_check.call(@raw, @rule)
+      end
+    end
+  end
+end

data/lib/schema_ferry/core/translator.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module SchemaFerry
+  module Core
+    class Translator
+      def initialize(config)
+        @config           = config
+        @column_converter = ColumnConverter.new(TypeMapper.new(config.global_type_overrides))
+        @enum_check       = (EnumCheckBuilder.new if config.enum_mode == :check)
+      end
+
+      def translate(raw_tables)
+        kept_tables  = raw_tables.reject { |t| @config.ignored_tables.include?(t[:name]) }
+        foreign_keys = kept_tables.to_h { |t| [t[:name], surviving_foreign_keys(t)] }
+        fk_columns   = collect_fk_columns(foreign_keys)
+
+        pg_tables = kept_tables.map do |raw|
+          TableConverter.new(
+            raw,
+            rule:             @config.table_rules[raw[:name]],
+            foreign_keys:     foreign_keys.fetch(raw[:name]),
+            fk_columns:       fk_columns.fetch(raw[:name], []),
+            column_converter: @column_converter,
+            enum_check:       @enum_check
+          ).convert
+        end
+
+        SchemafileRenderer.new.render(pg_tables)
+      end
+
+      private
+
+      def surviving_foreign_keys(raw)
+        ignored = @config.table_rules[raw[:name]].ignored_columns
+        raw[:foreign_keys]
+          .reject { |fk| @config.ignored_tables.include?(fk[:to_table]) }
+          .reject { |fk| ignored.include?(fk[:column]) }
+      end
+
+      # Columns on either side of a surviving foreign key must land in
+      # PostgreSQL's integer type family: a numeric(20, 0) column cannot
+      # reference a bigint key.
+      def collect_fk_columns(foreign_keys_by_table)
+        foreign_keys_by_table.each_with_object({}) do |(table_name, fks), acc|
+          fks.each do |fk|
+            (acc[table_name] ||= []) << fk[:column]
+            (acc[fk[:to_table]] ||= []) << fk[:primary_key]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/schema_ferry/{converter → core}/type_mapper.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module SchemaFerry
-  module Converter
+  module Core
     class TypeMapper
       # PG has no limit concept for text/binary/bigint/float; drop the
       # MySQL-derived limits (float's limit: 53 is DOUBLE's internal bit
@@ -36,7 +36,7 @@ module SchemaFerry
       def call(ar_type, options = {})
         unless KNOWN_TYPES.include?(ar_type)
           raise ConversionError, "Unknown MySQL AR type: #{ar_type.inspect}. " \
-                                 "Use map_type or map_column to specify a PostgreSQL type."
+                                 "Use map_type or column's map_type_to: to specify a PostgreSQL type."
         end
 
         pg_type  = @overrides.fetch(ar_type, ar_type)

data/lib/schema_ferry/errors.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module SchemaFerry
+  # Base class so the CLI can rescue every schema_ferry error in one clause.
   class Error               < StandardError; end
   class ConfigError         < Error; end
   class ConnectionError     < Error; end
@@ -8,4 +9,5 @@ module SchemaFerry
   class ConversionError     < Error; end
   class RidgepoleNotFoundError < Error; end
   class RidgepoleError < Error; end
+  class DropNotAllowedError < Error; end
 end

data/lib/schema_ferry/io/mysql_reader.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module SchemaFerry
+  module IO
+    class MysqlReader
+      def initialize(url)
+        @url = url
+      end
+
+      def read_all
+        with_connection do |conn|
+          conn.tables.sort.map { |table_name| read_table(conn, table_name) }
+        end
+      rescue ConnectionError
+        raise
+      rescue StandardError => e
+        raise ReadError, e.message
+      end
+
+      private
+
+      # Uses a fresh ConnectionHandler per call so the pool is fully isolated
+      # from ActiveRecord::Base and any host Rails app connections.
+      # AR 7.2+ banned anonymous AR subclasses, so we bypass that path entirely.
+      def with_connection
+        handler = ActiveRecord::ConnectionAdapters::ConnectionHandler.new
+        begin
+          pool = handler.establish_connection(@url, owner_name: "schema_ferry")
+          conn = pool.checkout
+          conn.verify!
+        rescue ActiveRecord::ActiveRecordError => e
+          raise ConnectionError, e.message
+        end
+
+        begin
+          yield conn
+        ensure
+          pool.checkin(conn)
+        end
+      ensure
+        handler&.clear_all_connections!
+      end
+
+      def read_table(conn, name)
+        {
+          name:         name,
+          primary_key:  conn.primary_key(name),
+          comment:      conn.table_comment(name),
+          columns:      serialize_columns(conn.columns(name)),
+          indexes:      serialize_indexes(conn.indexes(name)),
+          foreign_keys: serialize_foreign_keys(conn.foreign_keys(name))
+        }
+      end
+
+      def serialize_columns(columns)
+        fields = %i[name type sql_type limit precision scale null default default_function comment]
+        columns.map { |c| fields.to_h { |field| [field, c.public_send(field)] } }
+      end
+
+      def serialize_indexes(indexes)
+        indexes.map do |idx|
+          {
+            name:    idx.name,
+            columns: Array(idx.columns), # idx.columns can be an Array or a String
+            unique:  idx.unique,
+            using:   idx.using,
+            type:    idx.type,
+            lengths: idx.lengths,
+            orders:  idx.orders
+          }
+        end
+      end
+
+      def serialize_foreign_keys(fkeys)
+        fields = %i[from_table to_table column primary_key on_update on_delete name]
+        fkeys.map { |fk| fields.to_h { |field| [field, fk.public_send(field)] } }
+      end
+    end
+  end
+end

data/lib/schema_ferry/{target/ridgepole_runner.rb → io/postgres_writer.rb}

@@ -4,12 +4,12 @@ require "open3"
 require "tempfile"
 
 module SchemaFerry
-  module Target
+  module IO
     # Runs ridgepole as a subprocess rather than via Ridgepole::Client:
-    # the client calls ActiveRecord::Base.establish_connection and patches AR
-    # in-process, which would hijack a host Rails app's database connection.
-    # The tempfile exists because the CLI only reads the schema from -f FILE.
-    class RidgepoleRunner
+    # Ridgepole::Client calls ActiveRecord::Base.establish_connection and
+    # patches AR in-process, which would hijack a host Rails app's database
+    # connection.
+    class PostgresWriter
       def initialize(target_url)
         @target_url = target_url
       end
@@ -17,12 +17,17 @@ module SchemaFerry
       def run(schema_content, dry_run: false)
         bin = find_ridgepole!
 
+        # ridgepole only reads the schema from -f FILE.
         Tempfile.create(["schema_ferry_", ".rb"]) do |f|
           f.write(schema_content)
           f.flush
 
           cmd = [bin, "--apply", "-c", @target_url, "-f", f.path]
-          cmd << "--dry-run" if dry_run
+          if dry_run
+            cmd << "--dry-run"
+          else
+            cmd += ["--pre-query", "BEGIN", "--post-query", "COMMIT"]
+          end
 
           stdout, stderr, status = Open3.capture3(*cmd)
           raise RidgepoleError, [stdout, stderr].reject { |s| s.strip.empty? }.join("\n") unless status.success?

data/lib/schema_ferry/pipeline.rb

@@ -2,22 +2,39 @@
 
 module SchemaFerry
   class Pipeline
+    include Support::DropDetectable
+
     def initialize(config)
       @config = config
     end
 
     def dry_run
-      Target::RidgepoleRunner.new(@config.target_url).run(schemafile, dry_run: true)
+      write(build_schemafile, dry_run: true)
+    end
+
+    def apply!(allow_drops: true)
+      schemafile = build_schemafile
+
+      unless allow_drops
+        drops = detect_drops(write(schemafile, dry_run: true))
+        unless drops.empty?
+          raise DropNotAllowedError,
+                "refused: the diff contains drop(s):\n#{drops.join("\n")}"
+        end
+      end
+
+      write(schemafile, dry_run: false)
     end
 
-    def apply!
-      Target::RidgepoleRunner.new(@config.target_url).run(schemafile, dry_run: false)
+    private
+
+    def build_schemafile
+      IO::MysqlReader.new(@config.source_url).read_all
+        .then { |mysql_tables| Core::Translator.new(@config).translate(mysql_tables) }
     end
 
-    def schemafile
-      mysql_tables = Source::MysqlReader.new(@config.source_url).read_all
-      pg_tables    = Converter::SchemaConverter.new(@config).convert(mysql_tables)
-      Target::SchemafileRenderer.new.render(pg_tables)
+    def write(schemafile, dry_run:)
+      IO::PostgresWriter.new(@config.target_url).run(schemafile, dry_run: dry_run)
     end
   end
 end

data/lib/schema_ferry/support/drop_detectable.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module SchemaFerry
+  module Support
+    module DropDetectable
+      DROP_LINE = /^(?:drop_table|remove_column|remove_index|remove_foreign_key|
+                      remove_check_constraint|remove_exclusion_constraint|
+                      remove_unique_constraint)\(.*\)$/x
+
+      private
+
+      def detect_drops(ddl)
+        ddl.each_line.map(&:chomp).grep(DROP_LINE)
+      end
+    end
+  end
+end

data/lib/schema_ferry/support/warnings.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module SchemaFerry
+  module Support
+    module Warnings
+      private
+
+      def emit_warning(message)
+        Kernel.warn("[schema_ferry] #{message}")
+      end
+    end
+  end
+end

data/lib/schema_ferry/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SchemaFerry
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/schema_ferry.rb

@@ -4,23 +4,23 @@ require "active_record"
 
 require_relative "schema_ferry/version"
 require_relative "schema_ferry/errors"
-require_relative "schema_ferry/warnings"
-require_relative "schema_ferry/schema_model"
-require_relative "schema_ferry/dsl/table_rule"
-require_relative "schema_ferry/dsl/config"
-require_relative "schema_ferry/source/connection_registry"
-require_relative "schema_ferry/source/mysql_reader"
-require_relative "schema_ferry/converter/type_mapper"
-require_relative "schema_ferry/converter/identifier_shortener"
-require_relative "schema_ferry/converter/column_converter"
-require_relative "schema_ferry/converter/enum_check_builder"
-require_relative "schema_ferry/converter/schema_converter"
-require_relative "schema_ferry/target/schemafile_renderer"
-require_relative "schema_ferry/target/ridgepole_runner"
+require_relative "schema_ferry/support/warnings"
+require_relative "schema_ferry/support/drop_detectable"
+require_relative "schema_ferry/config"
+require_relative "schema_ferry/io/mysql_reader"
+require_relative "schema_ferry/core/schema_model"
+require_relative "schema_ferry/core/type_mapper"
+require_relative "schema_ferry/core/identifier_shortenable"
+require_relative "schema_ferry/core/column_converter"
+require_relative "schema_ferry/core/enum_check_builder"
+require_relative "schema_ferry/core/table_converter"
+require_relative "schema_ferry/core/translator"
+require_relative "schema_ferry/core/schemafile_renderer"
+require_relative "schema_ferry/io/postgres_writer"
 require_relative "schema_ferry/pipeline"
 
 module SchemaFerry
   def self.define(&)
-    Pipeline.new(DSL::Config.build(&))
+    Pipeline.new(Config.build(&))
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: schema_ferry
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - kyuuri1791
@@ -65,6 +65,11 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+description: schema_ferry keeps a PostgreSQL schema in sync with a MySQL source during
+  a gradual migration. It reads the MySQL schema through ActiveRecord, converts it
+  to PostgreSQL equivalents — with a declarative DSL for type mappings, enum handling,
+  and per-table overrides — and applies only the diff, idempotently, via ridgepole.
+  Ships a Ruby API and a cron-friendly CLI.
 executables:
 - schema_ferry
 extensions: []
@@ -76,28 +81,29 @@ files:
 - exe/schema_ferry
 - lib/schema_ferry.rb
 - lib/schema_ferry/cli.rb
-- lib/schema_ferry/converter/column_converter.rb
-- lib/schema_ferry/converter/enum_check_builder.rb
-- lib/schema_ferry/converter/identifier_shortener.rb
-- lib/schema_ferry/converter/schema_converter.rb
-- lib/schema_ferry/converter/type_mapper.rb
-- lib/schema_ferry/dsl/config.rb
-- lib/schema_ferry/dsl/table_rule.rb
+- lib/schema_ferry/config.rb
+- lib/schema_ferry/core/column_converter.rb
+- lib/schema_ferry/core/enum_check_builder.rb
+- lib/schema_ferry/core/identifier_shortenable.rb
+- lib/schema_ferry/core/schema_model.rb
+- lib/schema_ferry/core/schemafile_renderer.rb
+- lib/schema_ferry/core/table_converter.rb
+- lib/schema_ferry/core/translator.rb
+- lib/schema_ferry/core/type_mapper.rb
 - lib/schema_ferry/errors.rb
+- lib/schema_ferry/io/mysql_reader.rb
+- lib/schema_ferry/io/postgres_writer.rb
 - lib/schema_ferry/pipeline.rb
-- lib/schema_ferry/schema_model.rb
-- lib/schema_ferry/source/connection_registry.rb
-- lib/schema_ferry/source/mysql_reader.rb
-- lib/schema_ferry/target/ridgepole_runner.rb
-- lib/schema_ferry/target/schemafile_renderer.rb
+- lib/schema_ferry/support/drop_detectable.rb
+- lib/schema_ferry/support/warnings.rb
 - lib/schema_ferry/version.rb
-- lib/schema_ferry/warnings.rb
 homepage: https://github.com/kyuuri1791/schema_ferry
 licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/kyuuri1791/schema_ferry
   source_code_uri: https://github.com/kyuuri1791/schema_ferry
+  changelog_uri: https://github.com/kyuuri1791/schema_ferry/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths: