convergence 1.0.6 → 1.2.0

data/lib/convergence/command/apply.rb

@@ -8,40 +8,69 @@ require 'convergence/diff'
 class Convergence::Command::Apply < Convergence::Command
   def execute
     current_dir_path = Pathname.new(@opts[:input]).realpath.dirname
-    input_tables = Convergence::DSL.parse(File.open(@opts[:input]).read, current_dir_path)
+    dsl = Convergence::DSL.parse_dsl(File.open(@opts[:input]).read, current_dir_path)
     current_tables = dumper.dump
-    execute_sql(input_tables, current_tables)
+    execute_sql(dsl.tables, current_tables, dsl.raw_sqls)
   end
 
-  def generate_sql(input_tables, current_tables)
+  def generate_sql(input_tables, current_tables, raw_sqls = [])
     current_tables_with_full_option =
       Convergence::DefaultParameter.append_database_default_parameter(current_tables, database_adapter)
     input_tables_with_full_option =
       Convergence::DefaultParameter.append_database_default_parameter(input_tables, database_adapter)
-    delta = Convergence::Diff.new.diff(current_tables_with_full_option, input_tables_with_full_option)
-    sql_generator.generate(input_tables_with_full_option, delta, current_tables_with_full_option)
+    delta = Convergence::Diff
+      .new(ignore_auto_increment: @opts[:ignore_auto_increment])
+      .diff(current_tables_with_full_option, input_tables_with_full_option)
+    sql = sql_generator.generate(
+      input_tables_with_full_option,
+      delta,
+      current_tables_with_full_option,
+      safe_migration: @opts[:safe_migration]
+    )
+    append_raw_sqls(sql, raw_sqls)
   end
 
   private
 
+  def append_raw_sqls(sql, raw_sqls)
+    return sql if raw_sqls.empty?
+    raw_sql_block = raw_sqls.map { |q| q.strip.end_with?(';') ? q.strip : "#{q.strip};" }.join("\n")
+    [sql, raw_sql_block].reject(&:empty?).join("\n")
+  end
+
   def sql_generator
     @sql_generator ||= case database_adapter
                        when 'mysql', 'mysql2'
                          require 'convergence/sql_generator/mysql_generator'
                          SQLGenerator::MysqlGenerator.new
+                       when 'postgresql', 'postgres', 'pg'
+                         require 'convergence/sql_generator/postgres_generator'
+                         SQLGenerator::PostgresGenerator.new
+                       when 'sqlite3', 'sqlite'
+                         require 'convergence/sql_generator/sqlite_generator'
+                         SQLGenerator::SqliteGenerator.new
                        else
                          fail NotImplementedError.new('unknown database adapter')
                        end
   end
 
-  def execute_sql(input_tables, current_tables)
-    sql = generate_sql(input_tables, current_tables)
-    unless sql.strip.empty?
-      sql = <<-SQL
+  def wrap_with_constraint_pragma(sql)
+    case database_adapter
+    when 'mysql', 'mysql2'
+      <<-SQL
 SET FOREIGN_KEY_CHECKS=0;
       #{sql}
 SET FOREIGN_KEY_CHECKS=1;
       SQL
+    else
+      sql
+    end
+  end
+
+  def execute_sql(input_tables, current_tables, raw_sqls = [])
+    sql = generate_sql(input_tables, current_tables, raw_sqls)
+    unless sql.strip.empty?
+      sql = wrap_with_constraint_pragma(sql)
     end
     sql.split(';').each do |q2|
       q = q2.strip

data/lib/convergence/command/dryrun.rb

@@ -8,11 +8,11 @@ require 'convergence/pretty_diff'
 class Convergence::Command::Dryrun < Convergence::Command
   def execute
     current_dir_path = Pathname.new(@opts[:input]).realpath.dirname
-    input_tables = Convergence::DSL.parse(File.open(@opts[:input]).read, current_dir_path)
+    dsl = Convergence::DSL.parse_dsl(File.open(@opts[:input]).read, current_dir_path)
     current_tables = dumper.dump
     # -- maybe it's redundant output
-    # output_diff(input_tables, current_tables)
-    output_sql(input_tables, current_tables)
+    # output_diff(dsl.tables, current_tables)
+    output_sql(dsl.tables, current_tables, dsl.raw_sqls)
   end
 
   private
@@ -30,10 +30,10 @@ class Convergence::Command::Dryrun < Convergence::Command
     msg
   end
 
-  def output_sql(input_tables, current_tables)
+  def output_sql(input_tables, current_tables, raw_sqls = [])
     msg = Convergence::Command::Apply
       .new(@opts, config: @config)
-      .generate_sql(input_tables, current_tables)
+      .generate_sql(input_tables, current_tables, raw_sqls)
       .split("\n")
       .map { |v| '# ' + v }
       .join("\n")

data/lib/convergence/command/export.rb

@@ -5,8 +5,26 @@ require 'convergence/default_parameter'
 class Convergence::Command::Export < Convergence::Command
   def execute
     tables = Convergence::DefaultParameter.remove_database_default_parameter(dumper.dump, database_adapter)
-    msg = Convergence::Dumper.new.dump_dsl(tables)
+    msg = if @opts[:dump_rails_migration]
+            dump_rails_migration(tables)
+          else
+            Convergence::Dumper.new.dump_dsl(tables)
+          end
     logger.output(msg)
     msg
   end
+
+  private
+
+  def dump_rails_migration(tables)
+    filename = @opts[:filename] || 'convergence_migration'
+    class_name = camelize(filename)
+    migration_filename = "#{Time.now.strftime('%Y%m%d%H%M%S')}_#{filename}.rb"
+    body = Convergence::Dumper.new.dump_rails_migration(tables, class_name)
+    "# Filename: #{migration_filename}\n#{body}"
+  end
+
+  def camelize(name)
+    name.to_s.split(/[_-]/).reject(&:empty?).map { |part| part[0].upcase + part[1..].to_s }.join
+  end
 end

data/lib/convergence/command.rb

@@ -27,6 +27,12 @@ class Convergence::Command
                 when 'mysql', 'mysql2'
                   require 'convergence/dumper/mysql_schema_dumper'
                   Convergence::Dumper::MysqlSchemaDumper.new(connector)
+                when 'postgresql', 'postgres', 'pg'
+                  require 'convergence/dumper/postgres_schema_dumper'
+                  Convergence::Dumper::PostgresSchemaDumper.new(connector)
+                when 'sqlite3', 'sqlite'
+                  require 'convergence/dumper/sqlite_schema_dumper'
+                  Convergence::Dumper::SqliteSchemaDumper.new(connector)
                 else
                   fail NotImplementedError.new('unknown database adapter')
                 end

data/lib/convergence/database_connector/postgres_connector.rb

@@ -0,0 +1,25 @@
+require 'pg'
+require 'convergence/database_connector'
+
+class Convergence::DatabaseConnector::PostgresConnector
+  attr_reader :config
+
+  def initialize(config)
+    @config = config
+  end
+
+  def client(database_name = @config.database)
+    @clients ||= {}
+    @clients[database_name] ||= PG::Connection.new(
+      host: @config.host,
+      port: @config.port,
+      user: @config.username,
+      password: @config.password,
+      dbname: database_name
+    )
+  end
+
+  def schema_client
+    client
+  end
+end

data/lib/convergence/database_connector/sqlite_connector.rb

@@ -0,0 +1,22 @@
+require 'sqlite3'
+require 'convergence/database_connector'
+
+class Convergence::DatabaseConnector::SqliteConnector
+  attr_reader :config
+
+  def initialize(config)
+    @config = config
+  end
+
+  def client(database_name = @config.database)
+    @clients ||= {}
+    @clients[database_name] ||= SQLite3::Database.new(database_name).tap do |db|
+      db.results_as_hash = true
+      db.execute('PRAGMA foreign_keys = ON')
+    end
+  end
+
+  def schema_client
+    client
+  end
+end

data/lib/convergence/database_connector.rb

@@ -7,6 +7,12 @@ class Convergence::DatabaseConnector
       when 'mysql', 'mysql2'
         require 'convergence/database_connector/mysql_connector'
         Convergence::DatabaseConnector::MysqlConnector.new(config)
+      when 'postgresql', 'postgres', 'pg'
+        require 'convergence/database_connector/postgres_connector'
+        Convergence::DatabaseConnector::PostgresConnector.new(config)
+      when 'sqlite3', 'sqlite'
+        require 'convergence/database_connector/sqlite_connector'
+        Convergence::DatabaseConnector::SqliteConnector.new(config)
       else
         fail NotImplementedError.new("#{config.adapter} not supported yet")
       end

data/lib/convergence/default_parameter/mysql_default_parameter.rb

@@ -51,7 +51,7 @@ class Convergence::DefaultParameter::MysqlDefaultParameter
   DEFAULT_COLUMN_PARAMETERS = {
     null: false
   }
-  TEXT_TYPE = [:varchar, :char, :tiny_text, :text, :mediumtext, :longtext]
+  TEXT_TYPE = [:varchar, :char, :tiny_text, :text, :mediumtext, :longtext, :enum, :set]
   DEFAULT_COLUMN_TYPE_PARAMETERS = {
     tinyint: {
       limit: 4

data/lib/convergence/default_parameter/postgres_default_parameter.rb

@@ -0,0 +1,57 @@
+require 'convergence/default_parameter'
+
+class Convergence::DefaultParameter::PostgresDefaultParameter
+  DEFAULT_COLUMN_PARAMETERS = {
+    null: false
+  }
+  DEFAULT_INDEX_PARAMETERS = { type: 'btree', unique: false }
+
+  def initialize
+  end
+
+  def remove_default_parameter(table)
+    remove_column_default_parameter(table)
+    remove_index_default_parameter(table)
+    table
+  end
+
+  def append_default_parameter(table)
+    append_column_default_parameter(table)
+    append_index_default_parameter(table)
+    table
+  end
+
+  private
+
+  def remove_column_default_parameter(table)
+    table.columns.each do |_column_name, column|
+      DEFAULT_COLUMN_PARAMETERS.each do |k, v|
+        if !column.options[k].nil? && column.options[k].to_s.downcase == v.to_s.downcase
+          column.options.delete(k)
+        end
+      end
+    end
+  end
+
+  def remove_index_default_parameter(table)
+    table.indexes.each do |_, va|
+      va.options.each do |k, v|
+        if !DEFAULT_INDEX_PARAMETERS[k].nil? && DEFAULT_INDEX_PARAMETERS[k].to_s.downcase == v.to_s.downcase
+          va.options.delete(k)
+        end
+      end
+    end
+  end
+
+  def append_column_default_parameter(table)
+    table.columns.each do |_column_name, column|
+      column.options = DEFAULT_COLUMN_PARAMETERS.merge(column.options)
+    end
+  end
+
+  def append_index_default_parameter(table)
+    table.indexes.each do |_column_name, column|
+      column.options = DEFAULT_INDEX_PARAMETERS.merge(column.options)
+    end
+  end
+end

data/lib/convergence/default_parameter/sqlite_default_parameter.rb

@@ -0,0 +1,57 @@
+require 'convergence/default_parameter'
+
+class Convergence::DefaultParameter::SqliteDefaultParameter
+  DEFAULT_COLUMN_PARAMETERS = {
+    null: false
+  }
+  DEFAULT_INDEX_PARAMETERS = { unique: false }
+
+  def initialize
+  end
+
+  def remove_default_parameter(table)
+    remove_column_default_parameter(table)
+    remove_index_default_parameter(table)
+    table
+  end
+
+  def append_default_parameter(table)
+    append_column_default_parameter(table)
+    append_index_default_parameter(table)
+    table
+  end
+
+  private
+
+  def remove_column_default_parameter(table)
+    table.columns.each do |_column_name, column|
+      DEFAULT_COLUMN_PARAMETERS.each do |k, v|
+        if !column.options[k].nil? && column.options[k].to_s.downcase == v.to_s.downcase
+          column.options.delete(k)
+        end
+      end
+    end
+  end
+
+  def remove_index_default_parameter(table)
+    table.indexes.each do |_, va|
+      va.options.each do |k, v|
+        if !DEFAULT_INDEX_PARAMETERS[k].nil? && DEFAULT_INDEX_PARAMETERS[k].to_s.downcase == v.to_s.downcase
+          va.options.delete(k)
+        end
+      end
+    end
+  end
+
+  def append_column_default_parameter(table)
+    table.columns.each do |_column_name, column|
+      column.options = DEFAULT_COLUMN_PARAMETERS.merge(column.options)
+    end
+  end
+
+  def append_index_default_parameter(table)
+    table.indexes.each do |_column_name, column|
+      column.options = DEFAULT_INDEX_PARAMETERS.merge(column.options)
+    end
+  end
+end

data/lib/convergence/default_parameter.rb

@@ -4,6 +4,12 @@ class Convergence::DefaultParameter
     when 'mysql', 'mysql2'
       require 'convergence/default_parameter/mysql_default_parameter'
       @parameter_klass = Convergence::DefaultParameter::MysqlDefaultParameter.new
+    when 'postgresql', 'postgres', 'pg'
+      require 'convergence/default_parameter/postgres_default_parameter'
+      @parameter_klass = Convergence::DefaultParameter::PostgresDefaultParameter.new
+    when 'sqlite3', 'sqlite'
+      require 'convergence/default_parameter/sqlite_default_parameter'
+      @parameter_klass = Convergence::DefaultParameter::SqliteDefaultParameter.new
     else
       fail NotImplementedError.new("unknown adapter #{config.adapter}.")
     end

data/lib/convergence/diff.rb

@@ -2,11 +2,18 @@ require 'diff/lcs'
 
 class Convergence::Diff
   CASE_SENSITIVE_TABLE_OPTIONS = %i(comment)
-  CASE_SENSITIVE_COLUMNS = %i(default comment)
+  CASE_SENSITIVE_COLUMNS = %i(default comment values)
+
+  def initialize(ignore_auto_increment: false)
+    @ignore_auto_increment = ignore_auto_increment
+  end
 
   def diff(from_database, to_database)
     delta = {}
     from_database = {} if from_database.nil?
+    delta[:rename_table] = scan_rename_table(from_database, to_database)
+    from_database = from_database.reject { |name, _| delta[:rename_table].key?(name) }
+    to_database = to_database.reject { |name, _| delta[:rename_table].value?(name) }
     delta[:add_table] = scan_add_table(from_database, to_database)
     delta[:remove_table] = scan_remove_table(from_database, to_database)
     change_table = scan_change_table(from_database, to_database)
@@ -20,8 +27,11 @@ class Convergence::Diff
     from = from_table.dup
     to = to_table.dup
     delta = {}
+    delta[:rename_column] = scan_rename_column(from, to)
+    from.columns = from.columns.reject { |name, _| delta[:rename_column].key?(name) }
+    to.columns = to.columns.reject { |name, _| delta[:rename_column].value?(name) }
     delta[:remove_column] = scan_remove_column(from, to)
-    return delta if removed_all_columns?(from, delta)
+    return delta if removed_all_columns?(from_table, delta)
     delta[:add_column] = scan_add_column(from, to)
     delta[:change_column] = scan_change_column(from, to)
     scan_change_order_column(from, to, delta)
@@ -35,6 +45,36 @@ class Convergence::Diff
 
   private
 
+  def scan_rename_table(from, to)
+    renames = {}
+    to.each do |new_name, to_table|
+      old_name = to_table.renamed_from
+      next if old_name.nil?
+      next unless from.key?(old_name)
+      next if from.key?(new_name)
+      if renames.key?(old_name)
+        fail ArgumentError.new("renamed_from '#{old_name}' is specified for multiple tables")
+      end
+      renames[old_name] = new_name
+    end
+    renames
+  end
+
+  def scan_rename_column(from, to)
+    renames = {}
+    to.columns.each do |new_name, to_column|
+      old_name = to_column.renamed_from
+      next if old_name.nil?
+      next unless from.columns.key?(old_name)
+      next if from.columns.key?(new_name)
+      if renames.key?(old_name)
+        fail ArgumentError.new("#{to.table_name}: renamed_from '#{old_name}' is specified for multiple columns")
+      end
+      renames[old_name] = new_name
+    end
+    renames
+  end
+
   def scan_add_table(from, to)
     to.reject { |table_name, _| from.map { |k, _| k }.include?(table_name) }
   end
@@ -76,14 +116,16 @@ class Convergence::Diff
       .map do |column_name, from_column|
         to_column = to.columns[column_name]
         if to_column
-          to_column_option_with_type = (from_column.options.map { |k, _v| { k => nil } }.reduce { |a, e| a.merge(e) } || {})
+          to_column_option_with_type = (from_column.options.map { |k, _v| [k, nil] }.to_h)
             .merge(to_column.options)
             .merge(type: to_column.type)
+            .tap { |opt| opt[:default] = opt[:default].call if opt[:default].is_a?(Proc) }
             .map { |k, v| [k, case_sensitive_column?(k) ? v&.to_s : v&.to_s&.downcase] }
             .to_a
           from_column_option_with_type = from_column
             .options
             .merge(type: from_column.type)
+            .tap { |opt| opt[:default] = opt[:default].call if opt[:default].is_a?(Proc) }
             .map { |k, v| [k, case_sensitive_column?(k) ? v&.to_s : v&.to_s&.downcase] }
             .to_a
           { column_name => Hash[(to_column_option_with_type - from_column_option_with_type)] }
@@ -175,6 +217,7 @@ class Convergence::Diff
   end
 
   def remove_auto_increment_option?(from_value, to_value)
+    return true if @ignore_auto_increment
     return true if from_value.nil? || to_value.nil?
     from_value >= to_value
   end

data/lib/convergence/dsl.rb

@@ -1,10 +1,11 @@
 require 'convergence/table'
 
 class Convergence::DSL
-  attr_accessor :tables, :current_dir_path
+  attr_accessor :tables, :raw_sqls, :current_dir_path
 
   def initialize
     @tables = {}
+    @raw_sqls = []
   end
 
   def create_table(table_name, options = {}, &block)
@@ -14,15 +15,29 @@ class Convergence::DSL
     table
   end
 
+  # Execute an arbitrary SQL statement that the DSL has no dedicated syntax for
+  # (e.g. triggers, stored procedures, data backfills). Unlike create_table,
+  # this is not diffed against the current schema: it runs every time the
+  # schema file is applied, so the SQL itself must be idempotent.
+  def execute(sql)
+    @raw_sqls << sql
+  end
+
   def include(path)
     next_dir_path = File.dirname("#{@current_dir_path}/#{path}")
-    @tables.merge!(Convergence::DSL.parse(File.open("#{current_dir_path}/#{path}").read, next_dir_path))
+    included = Convergence::DSL.parse_dsl(File.open("#{current_dir_path}/#{path}").read, next_dir_path)
+    @tables.merge!(included.tables)
+    @raw_sqls.concat(included.raw_sqls)
   end
 
   def self.parse(code, current_dir_path)
+    parse_dsl(code, current_dir_path).tables
+  end
+
+  def self.parse_dsl(code, current_dir_path)
     parser = new
     parser.current_dir_path = current_dir_path
     parser.instance_eval(code)
-    parser.tables
+    parser
   end
 end

data/lib/convergence/dumper/mysql_schema_dumper.rb

@@ -110,8 +110,8 @@ class Convergence::Dumper::MysqlSchemaDumper
     option.merge!(engine: table_option['ENGINE'])
     row_format = table_option['CREATE_OPTIONS'].scan(/=(.*)/).flatten[0] || table_option['ROW_FORMAT']
     option.merge!(row_format: row_format)
-    option.merge!(default_charset: table_option['CHARACTER_SET_NAME'])
-    option.merge!(collate: table_option['TABLE_COLLATION'])
+    option.merge!(default_charset: normalize_charset(table_option['CHARACTER_SET_NAME']))
+    option.merge!(collate: normalize_charset(table_option['TABLE_COLLATION']))
     option.merge!(comment: table_option['TABLE_COMMENT'])
     option.merge!(auto_increment: table_option['AUTO_INCREMENT']) if table_option['AUTO_INCREMENT']
     table.table_options = option
@@ -125,16 +125,19 @@ class Convergence::Dumper::MysqlSchemaDumper
   end
 
   def parse_column(column)
-    data_type = column['DATA_TYPE']
+    data_type = column['DATA_TYPE'].to_sym
     column_name = column['COLUMN_NAME']
     options = { null: column['IS_NULLABLE'] == 'YES' ? true : false }
-    options.merge!(default: column['COLUMN_DEFAULT']) unless column['COLUMN_DEFAULT'].nil?
-    options.merge!(character_set: column['CHARACTER_SET_NAME']) unless column['CHARACTER_SET_NAME'].nil?
-    options.merge!(collate: column['COLLATION_NAME']) unless column['COLLATION_NAME'].nil?
+    unless column['COLUMN_DEFAULT'].nil?
+      options.merge!(default: column_default_expression(data_type, column['COLUMN_DEFAULT']))
+    end
+    options.merge!(character_set: normalize_charset(column['CHARACTER_SET_NAME'])) unless column['CHARACTER_SET_NAME'].nil?
+    options.merge!(collate: normalize_charset(column['COLLATION_NAME'])) unless column['COLLATION_NAME'].nil?
     column_type = column['COLUMN_TYPE']
-    if data_type == 'enum' || data_type == 'set'
-      # TODO: implement
-    elsif data_type == 'decimal'
+    case data_type
+    when :enum, :set
+      options.merge!(values: parse_enum_or_set_values(column_type))
+    when :decimal
       precision, scale = column_type.scan(/\d+/)
       options.merge!(precision: precision, scale: scale)
     else
@@ -179,4 +182,29 @@ class Convergence::Dumper::MysqlSchemaDumper
       end
     end
   end
+
+  # column_type looks like "enum('a','b','c')" or "set('a','b','c')"
+  def parse_enum_or_set_values(column_type)
+    column_type[/\A(?:enum|set)\((.*)\)\z/i, 1]
+      .to_s
+      .scan(/'((?:[^']|'')*)'/)
+      .flatten
+      .map { |v| v.gsub("''", "'") }
+  end
+
+  # MySQL 8.0.24+ reports the `utf8` charset/collation as `utf8mb3` (e.g. `utf8mb3_general_ci`).
+  # Normalize it back to `utf8` so it still matches schemas written with the traditional name.
+  def normalize_charset(value)
+    return value if value.nil?
+    value.sub(/\Autf8mb3/, 'utf8')
+  end
+
+  def column_default_expression(data_type, value)
+    case [data_type, value]
+    when [:datetime, "CURRENT_TIMESTAMP"], [:timestamp, "CURRENT_TIMESTAMP"]
+      -> { value }
+    else
+      value
+    end
+  end
 end