convergence 1.0.6 → 1.2.0

data/lib/convergence/dumper/postgres_schema_dumper.rb

@@ -0,0 +1,193 @@
+require 'convergence/dumper'
+require 'convergence/table'
+
+class Convergence::Dumper::PostgresSchemaDumper
+  # Convergence's DSL type names follow MySQL terminology. Map PostgreSQL's
+  # internal type name (udt_name) back to the closest DSL type.
+  TYPE_MAPPING = {
+    'int2' => :smallint,
+    'int4' => :int,
+    'int8' => :bigint,
+    'varchar' => :varchar,
+    'bpchar' => :char,
+    'text' => :text,
+    'bytea' => :blob,
+    'timestamp' => :datetime,
+    'timestamptz' => :datetime,
+    'date' => :date,
+    'time' => :time,
+    'numeric' => :decimal,
+    'float4' => :float,
+    'float8' => :double,
+    'json' => :json,
+    'jsonb' => :json,
+    # Convergence has no native boolean DSL type; store it as tinyint(1), the same
+    # representation Convergence::Table#boolean produces for the MySQL adapter.
+    'bool' => :tinyint
+  }.freeze
+  DEFAULT_VALUE_PATTERN = /\A'(?<value>(?:[^']|'')*)'(?:::[\w. ]+)?\z/.freeze
+
+  def initialize(connector)
+    @connector = connector
+    @target_database = connector.config.database
+    @tables = {}
+  end
+
+  def dump
+    table_definitions = select_table_definitions
+    column_definitions = select_column_definitions.group_by { |r| r['table_name'] }
+    index_definitions = select_index_definitions.group_by { |r| r['table_name'] }
+    foreign_key_definitions = select_foreign_key_definitions.group_by { |r| r['table_name'] }
+    table_definitions.map { |r| r['table_name'] }.each do |table_name|
+      table = Convergence::Table.new(table_name)
+      parse_table_options(table, table_definitions.find { |r| r['table_name'] == table_name })
+      parse_columns(table, column_definitions[table_name])
+      parse_indexes(table, index_definitions[table_name])
+      parse_foreign_keys(table, foreign_key_definitions[table_name])
+      @tables[table_name] = table
+    end
+    @tables
+  end
+
+  private
+
+  def pg
+    @connector.schema_client
+  end
+
+  def select_table_definitions
+    pg.query("
+      SELECT
+        c.relname AS table_name,
+        obj_description(c.oid) AS table_comment
+      FROM pg_class c
+      INNER JOIN pg_namespace n ON n.oid = c.relnamespace
+      WHERE c.relkind = 'r' AND n.nspname = 'public'
+      ORDER BY c.relname
+    ").to_a
+  end
+
+  def select_column_definitions
+    pg.query("
+      SELECT
+        c.table_name, c.column_name, c.ordinal_position, c.udt_name, c.is_nullable, c.column_default,
+        c.character_maximum_length, c.numeric_precision, c.numeric_scale, c.is_identity,
+        col_description(pgc.oid, c.ordinal_position) AS column_comment
+      FROM information_schema.columns c
+      INNER JOIN pg_class pgc ON pgc.relname = c.table_name
+      INNER JOIN pg_namespace pgn ON pgn.oid = pgc.relnamespace AND pgn.nspname = c.table_schema
+      WHERE c.table_schema = 'public'
+      ORDER BY c.table_name, c.ordinal_position
+    ").to_a
+  end
+
+  def select_index_definitions
+    pg.query("
+      SELECT
+        t.relname AS table_name,
+        i.relname AS index_name,
+        a.attname AS column_name,
+        ix.indisunique AS is_unique,
+        ix.indisprimary AS is_primary,
+        array_position(ix.indkey, a.attnum) AS seq_in_index
+      FROM pg_index ix
+      INNER JOIN pg_class t ON t.oid = ix.indrelid
+      INNER JOIN pg_class i ON i.oid = ix.indexrelid
+      INNER JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(ix.indkey)
+      INNER JOIN pg_namespace n ON n.oid = t.relnamespace
+      WHERE n.nspname = 'public'
+      ORDER BY t.relname, i.relname, seq_in_index
+    ").to_a
+  end
+
+  def select_foreign_key_definitions
+    pg.query("
+      SELECT
+        tc.table_name,
+        tc.constraint_name,
+        kcu.column_name,
+        ccu.table_name AS referenced_table_name,
+        ccu.column_name AS referenced_column_name
+      FROM information_schema.table_constraints tc
+      INNER JOIN information_schema.key_column_usage kcu
+        ON kcu.constraint_name = tc.constraint_name AND kcu.table_schema = tc.table_schema
+      INNER JOIN information_schema.constraint_column_usage ccu
+        ON ccu.constraint_name = tc.constraint_name AND ccu.table_schema = tc.table_schema
+      WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_schema = 'public'
+    ").to_a
+  end
+
+  def parse_table_options(table, table_option)
+    option = {}
+    option.merge!(comment: table_option['table_comment']) unless table_option['table_comment'].nil?
+    table.table_options = option
+  end
+
+  def parse_columns(table, columns)
+    return if columns.nil?
+    columns.each do |column|
+      data_type, column_name, options = parse_column(column)
+      table.send(data_type, column_name, options)
+    end
+  end
+
+  def parse_column(column)
+    data_type = TYPE_MAPPING[column['udt_name']] || column['udt_name'].to_sym
+    column_name = column['column_name']
+    options = { null: column['is_nullable'] == 'YES' }
+    if column['is_identity'] == 'YES'
+      options.merge!(extra: 'auto_increment')
+    elsif !column['column_default'].nil?
+      options.merge!(default: column_default_expression(data_type, column['column_default']))
+    end
+    case data_type
+    when :decimal
+      options.merge!(precision: column['numeric_precision'], scale: column['numeric_scale'])
+    when :varchar, :char
+      options.merge!(limit: column['character_maximum_length']) unless column['character_maximum_length'].nil?
+    when :tinyint
+      options.merge!(limit: '1') if column['udt_name'] == 'bool'
+    end
+    options.merge!(comment: column['column_comment']) unless column['column_comment'].nil?
+    [data_type, column_name, options]
+  end
+
+  def column_default_expression(data_type, value)
+    return -> { 'CURRENT_TIMESTAMP' } if [:datetime].include?(data_type) && value.start_with?('CURRENT_TIMESTAMP')
+    return (value == 'true' ? '1' : '0') if data_type == :tinyint && %w(true false).include?(value)
+    match = DEFAULT_VALUE_PATTERN.match(value)
+    return match[:value].gsub("''", "'") if match
+    value
+  end
+
+  def parse_indexes(table, table_indexes)
+    return if table_indexes.nil?
+    table_indexes.group_by { |r| r['index_name'] }.each do |index_name, indexes|
+      columns = indexes.sort_by { |r| r['seq_in_index'].to_i }.map { |v| v['column_name'] }
+      if indexes.first['is_primary'] == 't'
+        columns.each do |column|
+          options = { primary_key: true }.merge(table.columns[column].options)
+          table.columns[column].options = options
+        end
+      else
+        options = { name: index_name, unique: indexes.first['is_unique'] == 't' }
+        table.index(columns, options)
+      end
+    end
+  end
+
+  def parse_foreign_keys(table, foreign_keys)
+    return if foreign_keys.nil?
+    foreign_keys.group_by { |r| r['constraint_name'] }.each do |constraint_name, rows|
+      columns = rows.map { |r| r['column_name'] }
+      to_table = rows.first['referenced_table_name']
+      to_columns = rows.map { |r| r['referenced_column_name'] }
+      options = {
+        reference: to_table,
+        reference_column: to_columns,
+        name: constraint_name
+      }
+      table.foreign_key(columns, options)
+    end
+  end
+end

data/lib/convergence/dumper/sqlite_schema_dumper.rb

@@ -0,0 +1,132 @@
+require 'convergence/dumper'
+require 'convergence/table'
+
+class Convergence::Dumper::SqliteSchemaDumper
+  DEFAULT_VALUE_PATTERN = /\A'(?<value>(?:[^']|'')*)'\z/.freeze
+  # SQLite has no CONSTRAINT-name introspection pragma; the name is only
+  # preserved in the original CREATE TABLE text.
+  FOREIGN_KEY_NAME_PATTERN = /CONSTRAINT\s+"?(?<name>\w+)"?\s+FOREIGN KEY\s*\(\s*"?(?<column>\w+)"?\s*\)/i.freeze
+
+  def initialize(connector)
+    @connector = connector
+    @tables = {}
+  end
+
+  def dump
+    table_names.each do |table_name|
+      table = Convergence::Table.new(table_name)
+      parse_columns(table, column_definitions(table_name))
+      parse_indexes(table, index_definitions(table_name))
+      parse_foreign_keys(table, table_name)
+      @tables[table_name] = table
+    end
+    @tables
+  end
+
+  private
+
+  def db
+    @connector.schema_client
+  end
+
+  def table_names
+    db.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").map { |r| r['name'] }
+  end
+
+  def table_sql(table_name)
+    row = db.execute('SELECT sql FROM sqlite_master WHERE type = ? AND name = ?', ['table', table_name]).first
+    row ? row['sql'].to_s : ''
+  end
+
+  def column_definitions(table_name)
+    db.execute(%(PRAGMA table_info("#{table_name}")))
+  end
+
+  def index_definitions(table_name)
+    # origin 'c' is an explicitly created index (CREATE [UNIQUE] INDEX). 'pk' and
+    # 'u' are auto-generated by a PRIMARY KEY/UNIQUE column constraint, already
+    # represented on the column itself, so they are excluded here.
+    db.execute(%(PRAGMA index_list("#{table_name}"))).select { |r| r['origin'] == 'c' }
+  end
+
+  def index_column_definitions(index_name)
+    db.execute(%(PRAGMA index_info("#{index_name}")))
+  end
+
+  def foreign_key_definitions(table_name)
+    db.execute(%(PRAGMA foreign_key_list("#{table_name}")))
+  end
+
+  def parse_columns(table, columns)
+    pk_columns = columns.select { |c| c['pk'].to_i > 0 }
+    single_integer_pk_cid = pk_columns.size == 1 ? pk_columns.first['cid'] : nil
+    columns.each do |column|
+      data_type, column_name, options = parse_column(column, single_integer_pk_cid)
+      table.send(data_type, column_name, options)
+    end
+  end
+
+  def parse_column(column, single_integer_pk_cid)
+    raw_type = column['type'].to_s
+    type_name = raw_type[/\A(\w+)/, 1].to_s
+    column_name = column['name']
+    is_auto_increment_pk = !single_integer_pk_cid.nil? && column['cid'] == single_integer_pk_cid && type_name.upcase == 'INTEGER'
+    data_type = if is_auto_increment_pk
+                  :int
+                elsif type_name.empty?
+                  :text
+                else
+                  type_name.downcase.to_sym
+                end
+    options = { null: column['notnull'].to_i.zero? }
+    if is_auto_increment_pk
+      # SQLite's `notnull` pragma column doesn't reflect the implicit NOT NULL
+      # of an INTEGER PRIMARY KEY (rowid alias), so it must be forced here.
+      options.merge!(primary_key: true, extra: 'auto_increment', null: false)
+    else
+      options.merge!(primary_key: true) if column['pk'].to_i > 0
+      options.merge!(default: column_default_expression(data_type, column['dflt_value'])) unless column['dflt_value'].nil?
+    end
+    case data_type
+    when :decimal
+      precision, scale = raw_type.scan(/\d+/)
+      options.merge!(precision: precision, scale: scale) if precision
+    else
+      limit = raw_type[/\((\d+)\)/, 1]
+      options.merge!(limit: limit) unless limit.nil?
+    end
+    [data_type, column_name, options]
+  end
+
+  def column_default_expression(data_type, value)
+    return -> { 'CURRENT_TIMESTAMP' } if data_type == :datetime && value.to_s.upcase == 'CURRENT_TIMESTAMP'
+    match = DEFAULT_VALUE_PATTERN.match(value)
+    return match[:value].gsub("''", "'") if match
+    value
+  end
+
+  def parse_indexes(table, indexes)
+    indexes.each do |index|
+      columns = index_column_definitions(index['name']).sort_by { |c| c['seqno'].to_i }.map { |c| c['name'] }
+      options = { name: index['name'], unique: index['unique'].to_i == 1 }
+      table.index(columns, options)
+    end
+  end
+
+  def parse_foreign_keys(table, table_name)
+    foreign_key_names = table_sql(table_name).scan(FOREIGN_KEY_NAME_PATTERN).to_h { |name, column| [column, name] }
+    foreign_key_definitions(table_name).group_by { |r| r['id'] }.each do |_id, rows|
+      sorted_rows = rows.sort_by { |r| r['seq'].to_i }
+      columns = sorted_rows.map { |r| r['from'] }
+      to_table = sorted_rows.first['table']
+      to_columns = sorted_rows.map { |r| r['to'] }
+      key_name = foreign_key_names[columns.first] || "fk_#{table.table_name}_#{columns.join('_')}"
+      options = {
+        reference: to_table,
+        reference_column: to_columns,
+        name: key_name
+      }
+      table.foreign_key(columns, options)
+    end
+  end
+end

data/lib/convergence/dumper.rb

@@ -1,4 +1,39 @@
 class Convergence::Dumper
+  # Convergence's DSL type names follow MySQL terminology; map them to the
+  # closest ActiveRecord migration column type.
+  RAILS_TYPE_MAPPING = {
+    tinyint: :integer,
+    smallint: :integer,
+    mediumint: :integer,
+    int: :integer,
+    bigint: :bigint,
+    float: :float,
+    double: :float,
+    decimal: :decimal,
+    char: :string,
+    varchar: :string,
+    tinyblob: :binary,
+    blob: :binary,
+    mediumblob: :binary,
+    longblob: :binary,
+    tinytext: :text,
+    text: :text,
+    mediumtext: :text,
+    longtext: :text,
+    date: :date,
+    time: :time,
+    datetime: :datetime,
+    timestamp: :datetime,
+    year: :integer,
+    json: :json,
+    # ActiveRecord has no native enum/set column type; fall back to string.
+    enum: :string,
+    set: :string
+  }.freeze
+  # Options that only make sense for convergence's own SQL generators, with no
+  # equivalent in an ActiveRecord migration's create_table DSL.
+  RAILS_UNSUPPORTED_COLUMN_OPTIONS = %i[character_set collate extra after values].freeze
+
   def dump_dsl(tables)
     tables.map do |_, table|
       dump_table_dsl(table)
@@ -24,8 +59,84 @@ class Convergence::Dumper
     dsl
   end
 
+  def dump_rails_migration(tables, class_name, rails_version: '7.0')
+    dsl = "class #{class_name} < ActiveRecord::Migration[#{rails_version}]\n"
+    dsl += "  def change\n"
+    dsl += tables.map { |_, table| dump_table_rails_migration(table) }.join("\n\n")
+    dsl += "\n  end\n"
+    dsl += 'end'
+    dsl
+  end
+
+  def dump_table_rails_migration(table)
+    table_argument = [table.table_name.to_sym.inspect]
+    dsl = "    create_table #{table_argument.join(', ')} do |t|\n"
+    body_lines = rails_column_lines(table.columns)
+    body_lines += table.indexes.map { |_, index| dump_rails_index(index) }
+    body_lines += table.foreign_keys.map { |_, key| dump_rails_foreign_key(key) }
+    dsl += body_lines.map { |line| "      #{line}" }.join("\n")
+    dsl += "\n    end"
+    dsl
+  end
+
   private
 
+  # ActiveRecord's create_table implicitly adds an auto-incrementing `id`
+  # primary key, so an explicit column matching that shape is redundant.
+  def default_primary_key_column?(column_name, column)
+    column_name == 'id' &&
+      [:int, :bigint].include?(column.type) &&
+      column.options[:primary_key] &&
+      column.options[:extra].to_s.upcase.include?('AUTO_INCREMENT')
+  end
+
+  def rails_timestamp_column?(column)
+    column.type == :datetime && Convergence::Dumper::RAILS_UNSUPPORTED_COLUMN_OPTIONS.none? { |k| column.options.key?(k) }
+  end
+
+  def rails_column_lines(columns)
+    columns = columns.reject { |name, column| default_primary_key_column?(name, column) }
+    if %w[created_at updated_at].all? { |name| columns.key?(name) && rails_timestamp_column?(columns[name]) } &&
+       columns['created_at'].options[:null] == columns['updated_at'].options[:null]
+      remaining = columns.reject { |name, _| %w[created_at updated_at].include?(name) }
+      lines = remaining.map { |_, column| dump_rails_column(column) }
+      lines << (columns['created_at'].options[:null] ? 't.timestamps' : 't.timestamps null: false')
+      lines
+    else
+      columns.map { |_, column| dump_rails_column(column) }
+    end
+  end
+
+  def dump_rails_column(column)
+    argument = [column.column_name.to_sym.inspect]
+    options = column.options.reject { |k, _v| RAILS_UNSUPPORTED_COLUMN_OPTIONS.include?(k) }
+    if column.type == :tinyint && column.options[:limit].to_s == '1'
+      column_type = :boolean
+      options = options.reject { |k, _v| k == :limit }
+      options = options.merge(default: false) if options[:default].to_s == '0'
+      options = options.merge(default: true) if options[:default].to_s == '1'
+    else
+      column_type = RAILS_TYPE_MAPPING[column.type] || column.type
+    end
+    argument << options.map { |k, v| key_value_text(k, v) }
+    "t.#{column_type} #{argument.flatten.join(', ')}"
+  end
+
+  def dump_rails_index(index)
+    argument = [single_or_multiple_symbol(index.index_columns)]
+    options = index.options.select { |k, _v| %i[name unique].include?(k) }
+    argument << options.map { |k, v| key_value_text(k, v) }
+    "t.index #{argument.flatten.join(', ')}"
+  end
+
+  def dump_rails_foreign_key(foreign_key)
+    argument = [foreign_key.to_table.to_sym.inspect]
+    argument << "column: #{single_or_multiple_symbol(foreign_key.from_columns)}"
+    argument << "primary_key: #{single_or_multiple_symbol(foreign_key.to_columns)}" unless foreign_key.to_columns == ['id']
+    argument << key_value_text('name', foreign_key.key_name)
+    "t.foreign_key #{argument.join(', ')}"
+  end
+
   def dump_column(column)
     argument = [column.column_name.to_sym.inspect]
     case [column.type, column.options[:limit]]
@@ -72,6 +183,8 @@ class Convergence::Dumper
   def key_value_text(k, v)
     value = if v.to_s == 'true' || v.to_s == 'false' || v.to_s =~ /^\d+$/
               v
+            elsif v.is_a?(Proc)
+              %(-> { #{v.call.inspect} })
             else
               %(#{v.inspect})
             end

data/lib/convergence/sql_generator/mysql_generator.rb

@@ -13,11 +13,12 @@ class SQLGenerator::MysqlGenerator < SQLGenerator
 
   attr_reader :original_table
 
-  def generate(to_table, delta, original_table)
+  def generate(to_table, delta, original_table, safe_migration: false)
     @original_table = original_table
     sqls = []
+    sqls << rename_table_sqls(delta)
     sqls << change_table_sql(to_table, delta)
-    sqls << drop_table_sqls(delta)
+    sqls << (safe_migration ? [] : drop_table_sqls(delta))
     sqls << create_table_sqls(delta)
     sqls.reject!(&:empty?)
     sqls.join("\n")
@@ -25,6 +26,12 @@ class SQLGenerator::MysqlGenerator < SQLGenerator
 
   private
 
+  def rename_table_sqls(delta)
+    results = delta[:rename_table].map { |old_name, new_name| %(ALTER TABLE `#{old_name}` RENAME TO `#{new_name}`;) }
+    results << '' unless results.empty?
+    results
+  end
+
   # FIXME: multiple pk change not supported yet
   def change_table_sql(to_table, delta)
     change_table = delta[:change_table]
@@ -36,6 +43,9 @@ class SQLGenerator::MysqlGenerator < SQLGenerator
       table_delta[:remove_index].each do |index_name, _index|
         results << alter_remove_index_sql(table_name, index_name)
       end
+      table_delta[:rename_column].each do |old_name, new_name|
+        results << alter_rename_column_sql(table_name, old_name, new_name)
+      end
       unless table_delta[:remove_column].empty?
         results << alter_remove_columns_sql(table_name, table_delta[:remove_column].values)
       end
@@ -73,6 +83,10 @@ class SQLGenerator::MysqlGenerator < SQLGenerator
     sql
   end
 
+  def alter_rename_column_sql(table_name, old_name, new_name)
+    %(ALTER TABLE `#{table_name}` RENAME COLUMN `#{old_name}` TO `#{new_name}`;)
+  end
+
   def alter_change_column_sql(table_name, column_name, change_column_option, to_table)
     column = to_table[table_name].columns[column_name]
     column.options.merge!(after: change_column_option[:after]) unless change_column_option[:after].nil?
@@ -166,6 +180,9 @@ DROP TABLE `#{table_name}`;
   def create_column_sql(column, output_primary_key: false, output_auto_increment: true)
     sql = "`#{column.column_name}`"
     sql += " #{column.type}"
+    if [:enum, :set].include?(column.type)
+      sql += "(#{quote_enum_or_set_values(column.options[:values])})"
+    end
     sql += "(#{column.options[:limit]})" unless column.options[:limit].nil?
     if column.options[:precision] && column.options[:scale]
       sql += "(#{column.options[:precision]}, #{column.options[:scale]})"
@@ -188,7 +205,7 @@ DROP TABLE `#{table_name}`;
       sql += ' PRIMARY KEY'
     end
     if column.options[:default]
-      sql += " DEFAULT '#{column.options[:default]}'"
+      sql += " DEFAULT #{quote_default_expression(column.options[:default])}"
     end
     if column.options[:comment]
       sql += " COMMENT '#{column.options[:comment]}'"
@@ -244,4 +261,16 @@ DROP TABLE `#{table_name}`;
       end
     end.join(' ')
   end
+
+  def quote_enum_or_set_values(values)
+    Array(values).map { |v| "'#{v.to_s.gsub("'", "''")}'" }.join(',')
+  end
+
+  def quote_default_expression(value)
+    if value.is_a?(Proc)
+      value.call
+    else
+      %('#{value}')
+    end
+  end
 end