convergence 1.0.6 → 1.1.0

data/lib/convergence/command/dryrun.rb

@@ -8,11 +8,11 @@ require 'convergence/pretty_diff'
 class Convergence::Command::Dryrun < Convergence::Command
   def execute
     current_dir_path = Pathname.new(@opts[:input]).realpath.dirname
-    input_tables = Convergence::DSL.parse(File.open(@opts[:input]).read, current_dir_path)
+    dsl = Convergence::DSL.parse_dsl(File.open(@opts[:input]).read, current_dir_path)
     current_tables = dumper.dump
     # -- maybe it's redundant output
-    # output_diff(input_tables, current_tables)
-    output_sql(input_tables, current_tables)
+    # output_diff(dsl.tables, current_tables)
+    output_sql(dsl.tables, current_tables, dsl.raw_sqls)
   end
 
   private
@@ -30,10 +30,10 @@ class Convergence::Command::Dryrun < Convergence::Command
     msg
   end
 
-  def output_sql(input_tables, current_tables)
+  def output_sql(input_tables, current_tables, raw_sqls = [])
     msg = Convergence::Command::Apply
       .new(@opts, config: @config)
-      .generate_sql(input_tables, current_tables)
+      .generate_sql(input_tables, current_tables, raw_sqls)
       .split("\n")
       .map { |v| '# ' + v }
       .join("\n")

data/lib/convergence/command/export.rb

@@ -5,8 +5,26 @@ require 'convergence/default_parameter'
 class Convergence::Command::Export < Convergence::Command
   def execute
     tables = Convergence::DefaultParameter.remove_database_default_parameter(dumper.dump, database_adapter)
-    msg = Convergence::Dumper.new.dump_dsl(tables)
+    msg = if @opts[:dump_rails_migration]
+            dump_rails_migration(tables)
+          else
+            Convergence::Dumper.new.dump_dsl(tables)
+          end
     logger.output(msg)
     msg
   end
+
+  private
+
+  def dump_rails_migration(tables)
+    filename = @opts[:filename] || 'convergence_migration'
+    class_name = camelize(filename)
+    migration_filename = "#{Time.now.strftime('%Y%m%d%H%M%S')}_#{filename}.rb"
+    body = Convergence::Dumper.new.dump_rails_migration(tables, class_name)
+    "# Filename: #{migration_filename}\n#{body}"
+  end
+
+  def camelize(name)
+    name.to_s.split(/[_-]/).reject(&:empty?).map { |part| part[0].upcase + part[1..].to_s }.join
+  end
 end

data/lib/convergence/command.rb

@@ -27,6 +27,12 @@ class Convergence::Command
                 when 'mysql', 'mysql2'
                   require 'convergence/dumper/mysql_schema_dumper'
                   Convergence::Dumper::MysqlSchemaDumper.new(connector)
+                when 'postgresql', 'postgres', 'pg'
+                  require 'convergence/dumper/postgres_schema_dumper'
+                  Convergence::Dumper::PostgresSchemaDumper.new(connector)
+                when 'sqlite3', 'sqlite'
+                  require 'convergence/dumper/sqlite_schema_dumper'
+                  Convergence::Dumper::SqliteSchemaDumper.new(connector)
                 else
                   fail NotImplementedError.new('unknown database adapter')
                 end

data/lib/convergence/database_connector/postgres_connector.rb

@@ -0,0 +1,25 @@
+require 'pg'
+require 'convergence/database_connector'
+
+class Convergence::DatabaseConnector::PostgresConnector
+  attr_reader :config
+
+  def initialize(config)
+    @config = config
+  end
+
+  def client(database_name = @config.database)
+    @clients ||= {}
+    @clients[database_name] ||= PG::Connection.new(
+      host: @config.host,
+      port: @config.port,
+      user: @config.username,
+      password: @config.password,
+      dbname: database_name
+    )
+  end
+
+  def schema_client
+    client
+  end
+end

data/lib/convergence/database_connector/sqlite_connector.rb

@@ -0,0 +1,22 @@
+require 'sqlite3'
+require 'convergence/database_connector'
+
+class Convergence::DatabaseConnector::SqliteConnector
+  attr_reader :config
+
+  def initialize(config)
+    @config = config
+  end
+
+  def client(database_name = @config.database)
+    @clients ||= {}
+    @clients[database_name] ||= SQLite3::Database.new(database_name).tap do |db|
+      db.results_as_hash = true
+      db.execute('PRAGMA foreign_keys = ON')
+    end
+  end
+
+  def schema_client
+    client
+  end
+end

data/lib/convergence/database_connector.rb

@@ -7,6 +7,12 @@ class Convergence::DatabaseConnector
       when 'mysql', 'mysql2'
         require 'convergence/database_connector/mysql_connector'
         Convergence::DatabaseConnector::MysqlConnector.new(config)
+      when 'postgresql', 'postgres', 'pg'
+        require 'convergence/database_connector/postgres_connector'
+        Convergence::DatabaseConnector::PostgresConnector.new(config)
+      when 'sqlite3', 'sqlite'
+        require 'convergence/database_connector/sqlite_connector'
+        Convergence::DatabaseConnector::SqliteConnector.new(config)
       else
         fail NotImplementedError.new("#{config.adapter} not supported yet")
       end

data/lib/convergence/default_parameter/mysql_default_parameter.rb

@@ -51,7 +51,7 @@ class Convergence::DefaultParameter::MysqlDefaultParameter
   DEFAULT_COLUMN_PARAMETERS = {
     null: false
   }
-  TEXT_TYPE = [:varchar, :char, :tiny_text, :text, :mediumtext, :longtext]
+  TEXT_TYPE = [:varchar, :char, :tiny_text, :text, :mediumtext, :longtext, :enum, :set]
   DEFAULT_COLUMN_TYPE_PARAMETERS = {
     tinyint: {
       limit: 4

data/lib/convergence/default_parameter/postgres_default_parameter.rb

@@ -0,0 +1,57 @@
+require 'convergence/default_parameter'
+
+class Convergence::DefaultParameter::PostgresDefaultParameter
+  DEFAULT_COLUMN_PARAMETERS = {
+    null: false
+  }
+  DEFAULT_INDEX_PARAMETERS = { type: 'btree', unique: false }
+
+  def initialize
+  end
+
+  def remove_default_parameter(table)
+    remove_column_default_parameter(table)
+    remove_index_default_parameter(table)
+    table
+  end
+
+  def append_default_parameter(table)
+    append_column_default_parameter(table)
+    append_index_default_parameter(table)
+    table
+  end
+
+  private
+
+  def remove_column_default_parameter(table)
+    table.columns.each do |_column_name, column|
+      DEFAULT_COLUMN_PARAMETERS.each do |k, v|
+        if !column.options[k].nil? && column.options[k].to_s.downcase == v.to_s.downcase
+          column.options.delete(k)
+        end
+      end
+    end
+  end
+
+  def remove_index_default_parameter(table)
+    table.indexes.each do |_, va|
+      va.options.each do |k, v|
+        if !DEFAULT_INDEX_PARAMETERS[k].nil? && DEFAULT_INDEX_PARAMETERS[k].to_s.downcase == v.to_s.downcase
+          va.options.delete(k)
+        end
+      end
+    end
+  end
+
+  def append_column_default_parameter(table)
+    table.columns.each do |_column_name, column|
+      column.options = DEFAULT_COLUMN_PARAMETERS.merge(column.options)
+    end
+  end
+
+  def append_index_default_parameter(table)
+    table.indexes.each do |_column_name, column|
+      column.options = DEFAULT_INDEX_PARAMETERS.merge(column.options)
+    end
+  end
+end

data/lib/convergence/default_parameter/sqlite_default_parameter.rb

@@ -0,0 +1,57 @@
+require 'convergence/default_parameter'
+
+class Convergence::DefaultParameter::SqliteDefaultParameter
+  DEFAULT_COLUMN_PARAMETERS = {
+    null: false
+  }
+  DEFAULT_INDEX_PARAMETERS = { unique: false }
+
+  def initialize
+  end
+
+  def remove_default_parameter(table)
+    remove_column_default_parameter(table)
+    remove_index_default_parameter(table)
+    table
+  end
+
+  def append_default_parameter(table)
+    append_column_default_parameter(table)
+    append_index_default_parameter(table)
+    table
+  end
+
+  private
+
+  def remove_column_default_parameter(table)
+    table.columns.each do |_column_name, column|
+      DEFAULT_COLUMN_PARAMETERS.each do |k, v|
+        if !column.options[k].nil? && column.options[k].to_s.downcase == v.to_s.downcase
+          column.options.delete(k)
+        end
+      end
+    end
+  end
+
+  def remove_index_default_parameter(table)
+    table.indexes.each do |_, va|
+      va.options.each do |k, v|
+        if !DEFAULT_INDEX_PARAMETERS[k].nil? && DEFAULT_INDEX_PARAMETERS[k].to_s.downcase == v.to_s.downcase
+          va.options.delete(k)
+        end
+      end
+    end
+  end
+
+  def append_column_default_parameter(table)
+    table.columns.each do |_column_name, column|
+      column.options = DEFAULT_COLUMN_PARAMETERS.merge(column.options)
+    end
+  end
+
+  def append_index_default_parameter(table)
+    table.indexes.each do |_column_name, column|
+      column.options = DEFAULT_INDEX_PARAMETERS.merge(column.options)
+    end
+  end
+end

data/lib/convergence/default_parameter.rb

@@ -4,6 +4,12 @@ class Convergence::DefaultParameter
     when 'mysql', 'mysql2'
       require 'convergence/default_parameter/mysql_default_parameter'
       @parameter_klass = Convergence::DefaultParameter::MysqlDefaultParameter.new
+    when 'postgresql', 'postgres', 'pg'
+      require 'convergence/default_parameter/postgres_default_parameter'
+      @parameter_klass = Convergence::DefaultParameter::PostgresDefaultParameter.new
+    when 'sqlite3', 'sqlite'
+      require 'convergence/default_parameter/sqlite_default_parameter'
+      @parameter_klass = Convergence::DefaultParameter::SqliteDefaultParameter.new
     else
       fail NotImplementedError.new("unknown adapter #{config.adapter}.")
     end

data/lib/convergence/diff.rb

@@ -2,7 +2,11 @@ require 'diff/lcs'
 
 class Convergence::Diff
   CASE_SENSITIVE_TABLE_OPTIONS = %i(comment)
-  CASE_SENSITIVE_COLUMNS = %i(default comment)
+  CASE_SENSITIVE_COLUMNS = %i(default comment values)
+
+  def initialize(ignore_auto_increment: false)
+    @ignore_auto_increment = ignore_auto_increment
+  end
 
   def diff(from_database, to_database)
     delta = {}
@@ -76,14 +80,16 @@ class Convergence::Diff
       .map do |column_name, from_column|
         to_column = to.columns[column_name]
         if to_column
-          to_column_option_with_type = (from_column.options.map { |k, _v| { k => nil } }.reduce { |a, e| a.merge(e) } || {})
+          to_column_option_with_type = (from_column.options.map { |k, _v| [k, nil] }.to_h)
             .merge(to_column.options)
             .merge(type: to_column.type)
+            .tap { |opt| opt[:default] = opt[:default].call if opt[:default].is_a?(Proc) }
             .map { |k, v| [k, case_sensitive_column?(k) ? v&.to_s : v&.to_s&.downcase] }
             .to_a
           from_column_option_with_type = from_column
             .options
             .merge(type: from_column.type)
+            .tap { |opt| opt[:default] = opt[:default].call if opt[:default].is_a?(Proc) }
             .map { |k, v| [k, case_sensitive_column?(k) ? v&.to_s : v&.to_s&.downcase] }
             .to_a
           { column_name => Hash[(to_column_option_with_type - from_column_option_with_type)] }
@@ -175,6 +181,7 @@ class Convergence::Diff
   end
 
   def remove_auto_increment_option?(from_value, to_value)
+    return true if @ignore_auto_increment
     return true if from_value.nil? || to_value.nil?
     from_value >= to_value
   end

data/lib/convergence/dsl.rb

@@ -1,10 +1,11 @@
 require 'convergence/table'
 
 class Convergence::DSL
-  attr_accessor :tables, :current_dir_path
+  attr_accessor :tables, :raw_sqls, :current_dir_path
 
   def initialize
     @tables = {}
+    @raw_sqls = []
   end
 
   def create_table(table_name, options = {}, &block)
@@ -14,15 +15,29 @@ class Convergence::DSL
     table
   end
 
+  # Execute an arbitrary SQL statement that the DSL has no dedicated syntax for
+  # (e.g. triggers, stored procedures, data backfills). Unlike create_table,
+  # this is not diffed against the current schema: it runs every time the
+  # schema file is applied, so the SQL itself must be idempotent.
+  def execute(sql)
+    @raw_sqls << sql
+  end
+
   def include(path)
     next_dir_path = File.dirname("#{@current_dir_path}/#{path}")
-    @tables.merge!(Convergence::DSL.parse(File.open("#{current_dir_path}/#{path}").read, next_dir_path))
+    included = Convergence::DSL.parse_dsl(File.open("#{current_dir_path}/#{path}").read, next_dir_path)
+    @tables.merge!(included.tables)
+    @raw_sqls.concat(included.raw_sqls)
   end
 
   def self.parse(code, current_dir_path)
+    parse_dsl(code, current_dir_path).tables
+  end
+
+  def self.parse_dsl(code, current_dir_path)
     parser = new
     parser.current_dir_path = current_dir_path
     parser.instance_eval(code)
-    parser.tables
+    parser
   end
 end

data/lib/convergence/dumper/mysql_schema_dumper.rb

@@ -110,8 +110,8 @@ class Convergence::Dumper::MysqlSchemaDumper
     option.merge!(engine: table_option['ENGINE'])
     row_format = table_option['CREATE_OPTIONS'].scan(/=(.*)/).flatten[0] || table_option['ROW_FORMAT']
     option.merge!(row_format: row_format)
-    option.merge!(default_charset: table_option['CHARACTER_SET_NAME'])
-    option.merge!(collate: table_option['TABLE_COLLATION'])
+    option.merge!(default_charset: normalize_charset(table_option['CHARACTER_SET_NAME']))
+    option.merge!(collate: normalize_charset(table_option['TABLE_COLLATION']))
     option.merge!(comment: table_option['TABLE_COMMENT'])
     option.merge!(auto_increment: table_option['AUTO_INCREMENT']) if table_option['AUTO_INCREMENT']
     table.table_options = option
@@ -125,16 +125,19 @@ class Convergence::Dumper::MysqlSchemaDumper
   end
 
   def parse_column(column)
-    data_type = column['DATA_TYPE']
+    data_type = column['DATA_TYPE'].to_sym
     column_name = column['COLUMN_NAME']
     options = { null: column['IS_NULLABLE'] == 'YES' ? true : false }
-    options.merge!(default: column['COLUMN_DEFAULT']) unless column['COLUMN_DEFAULT'].nil?
-    options.merge!(character_set: column['CHARACTER_SET_NAME']) unless column['CHARACTER_SET_NAME'].nil?
-    options.merge!(collate: column['COLLATION_NAME']) unless column['COLLATION_NAME'].nil?
+    unless column['COLUMN_DEFAULT'].nil?
+      options.merge!(default: column_default_expression(data_type, column['COLUMN_DEFAULT']))
+    end
+    options.merge!(character_set: normalize_charset(column['CHARACTER_SET_NAME'])) unless column['CHARACTER_SET_NAME'].nil?
+    options.merge!(collate: normalize_charset(column['COLLATION_NAME'])) unless column['COLLATION_NAME'].nil?
     column_type = column['COLUMN_TYPE']
-    if data_type == 'enum' || data_type == 'set'
-      # TODO: implement
-    elsif data_type == 'decimal'
+    case data_type
+    when :enum, :set
+      options.merge!(values: parse_enum_or_set_values(column_type))
+    when :decimal
       precision, scale = column_type.scan(/\d+/)
       options.merge!(precision: precision, scale: scale)
     else
@@ -179,4 +182,29 @@ class Convergence::Dumper::MysqlSchemaDumper
       end
     end
   end
+
+  # column_type looks like "enum('a','b','c')" or "set('a','b','c')"
+  def parse_enum_or_set_values(column_type)
+    column_type[/\A(?:enum|set)\((.*)\)\z/i, 1]
+      .to_s
+      .scan(/'((?:[^']|'')*)'/)
+      .flatten
+      .map { |v| v.gsub("''", "'") }
+  end
+
+  # MySQL 8.0.24+ reports the `utf8` charset/collation as `utf8mb3` (e.g. `utf8mb3_general_ci`).
+  # Normalize it back to `utf8` so it still matches schemas written with the traditional name.
+  def normalize_charset(value)
+    return value if value.nil?
+    value.sub(/\Autf8mb3/, 'utf8')
+  end
+
+  def column_default_expression(data_type, value)
+    case [data_type, value]
+    when [:datetime, "CURRENT_TIMESTAMP"], [:timestamp, "CURRENT_TIMESTAMP"]
+      -> { value }
+    else
+      value
+    end
+  end
 end

data/lib/convergence/dumper/postgres_schema_dumper.rb

@@ -0,0 +1,193 @@
+require 'convergence/dumper'
+require 'convergence/table'
+
+class Convergence::Dumper::PostgresSchemaDumper
+  # Convergence's DSL type names follow MySQL terminology. Map PostgreSQL's
+  # internal type name (udt_name) back to the closest DSL type.
+  TYPE_MAPPING = {
+    'int2' => :smallint,
+    'int4' => :int,
+    'int8' => :bigint,
+    'varchar' => :varchar,
+    'bpchar' => :char,
+    'text' => :text,
+    'bytea' => :blob,
+    'timestamp' => :datetime,
+    'timestamptz' => :datetime,
+    'date' => :date,
+    'time' => :time,
+    'numeric' => :decimal,
+    'float4' => :float,
+    'float8' => :double,
+    'json' => :json,
+    'jsonb' => :json,
+    # Convergence has no native boolean DSL type; store it as tinyint(1), the same
+    # representation Convergence::Table#boolean produces for the MySQL adapter.
+    'bool' => :tinyint
+  }.freeze
+  DEFAULT_VALUE_PATTERN = /\A'(?<value>(?:[^']|'')*)'(?:::[\w. ]+)?\z/.freeze
+
+  def initialize(connector)
+    @connector = connector
+    @target_database = connector.config.database
+    @tables = {}
+  end
+
+  def dump
+    table_definitions = select_table_definitions
+    column_definitions = select_column_definitions.group_by { |r| r['table_name'] }
+    index_definitions = select_index_definitions.group_by { |r| r['table_name'] }
+    foreign_key_definitions = select_foreign_key_definitions.group_by { |r| r['table_name'] }
+    table_definitions.map { |r| r['table_name'] }.each do |table_name|
+      table = Convergence::Table.new(table_name)
+      parse_table_options(table, table_definitions.find { |r| r['table_name'] == table_name })
+      parse_columns(table, column_definitions[table_name])
+      parse_indexes(table, index_definitions[table_name])
+      parse_foreign_keys(table, foreign_key_definitions[table_name])
+      @tables[table_name] = table
+    end
+    @tables
+  end
+
+  private
+
+  def pg
+    @connector.schema_client
+  end
+
+  def select_table_definitions
+    pg.query("
+      SELECT
+        c.relname AS table_name,
+        obj_description(c.oid) AS table_comment
+      FROM pg_class c
+      INNER JOIN pg_namespace n ON n.oid = c.relnamespace
+      WHERE c.relkind = 'r' AND n.nspname = 'public'
+      ORDER BY c.relname
+    ").to_a
+  end
+
+  def select_column_definitions
+    pg.query("
+      SELECT
+        c.table_name, c.column_name, c.ordinal_position, c.udt_name, c.is_nullable, c.column_default,
+        c.character_maximum_length, c.numeric_precision, c.numeric_scale, c.is_identity,
+        col_description(pgc.oid, c.ordinal_position) AS column_comment
+      FROM information_schema.columns c
+      INNER JOIN pg_class pgc ON pgc.relname = c.table_name
+      INNER JOIN pg_namespace pgn ON pgn.oid = pgc.relnamespace AND pgn.nspname = c.table_schema
+      WHERE c.table_schema = 'public'
+      ORDER BY c.table_name, c.ordinal_position
+    ").to_a
+  end
+
+  def select_index_definitions
+    pg.query("
+      SELECT
+        t.relname AS table_name,
+        i.relname AS index_name,
+        a.attname AS column_name,
+        ix.indisunique AS is_unique,
+        ix.indisprimary AS is_primary,
+        array_position(ix.indkey, a.attnum) AS seq_in_index
+      FROM pg_index ix
+      INNER JOIN pg_class t ON t.oid = ix.indrelid
+      INNER JOIN pg_class i ON i.oid = ix.indexrelid
+      INNER JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(ix.indkey)
+      INNER JOIN pg_namespace n ON n.oid = t.relnamespace
+      WHERE n.nspname = 'public'
+      ORDER BY t.relname, i.relname, seq_in_index
+    ").to_a
+  end
+
+  def select_foreign_key_definitions
+    pg.query("
+      SELECT
+        tc.table_name,
+        tc.constraint_name,
+        kcu.column_name,
+        ccu.table_name AS referenced_table_name,
+        ccu.column_name AS referenced_column_name
+      FROM information_schema.table_constraints tc
+      INNER JOIN information_schema.key_column_usage kcu
+        ON kcu.constraint_name = tc.constraint_name AND kcu.table_schema = tc.table_schema
+      INNER JOIN information_schema.constraint_column_usage ccu
+        ON ccu.constraint_name = tc.constraint_name AND ccu.table_schema = tc.table_schema
+      WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_schema = 'public'
+    ").to_a
+  end
+
+  def parse_table_options(table, table_option)
+    option = {}
+    option.merge!(comment: table_option['table_comment']) unless table_option['table_comment'].nil?
+    table.table_options = option
+  end
+
+  def parse_columns(table, columns)
+    return if columns.nil?
+    columns.each do |column|
+      data_type, column_name, options = parse_column(column)
+      table.send(data_type, column_name, options)
+    end
+  end
+
+  def parse_column(column)
+    data_type = TYPE_MAPPING[column['udt_name']] || column['udt_name'].to_sym
+    column_name = column['column_name']
+    options = { null: column['is_nullable'] == 'YES' }
+    if column['is_identity'] == 'YES'
+      options.merge!(extra: 'auto_increment')
+    elsif !column['column_default'].nil?
+      options.merge!(default: column_default_expression(data_type, column['column_default']))
+    end
+    case data_type
+    when :decimal
+      options.merge!(precision: column['numeric_precision'], scale: column['numeric_scale'])
+    when :varchar, :char
+      options.merge!(limit: column['character_maximum_length']) unless column['character_maximum_length'].nil?
+    when :tinyint
+      options.merge!(limit: '1') if column['udt_name'] == 'bool'
+    end
+    options.merge!(comment: column['column_comment']) unless column['column_comment'].nil?
+    [data_type, column_name, options]
+  end
+
+  def column_default_expression(data_type, value)
+    return -> { 'CURRENT_TIMESTAMP' } if [:datetime].include?(data_type) && value.start_with?('CURRENT_TIMESTAMP')
+    return (value == 'true' ? '1' : '0') if data_type == :tinyint && %w(true false).include?(value)
+    match = DEFAULT_VALUE_PATTERN.match(value)
+    return match[:value].gsub("''", "'") if match
+    value
+  end
+
+  def parse_indexes(table, table_indexes)
+    return if table_indexes.nil?
+    table_indexes.group_by { |r| r['index_name'] }.each do |index_name, indexes|
+      columns = indexes.sort_by { |r| r['seq_in_index'].to_i }.map { |v| v['column_name'] }
+      if indexes.first['is_primary'] == 't'
+        columns.each do |column|
+          options = { primary_key: true }.merge(table.columns[column].options)
+          table.columns[column].options = options
+        end
+      else
+        options = { name: index_name, unique: indexes.first['is_unique'] == 't' }
+        table.index(columns, options)
+      end
+    end
+  end
+
+  def parse_foreign_keys(table, foreign_keys)
+    return if foreign_keys.nil?
+    foreign_keys.group_by { |r| r['constraint_name'] }.each do |constraint_name, rows|
+      columns = rows.map { |r| r['column_name'] }
+      to_table = rows.first['referenced_table_name']
+      to_columns = rows.map { |r| r['referenced_column_name'] }
+      options = {
+        reference: to_table,
+        reference_column: to_columns,
+        name: constraint_name
+      }
+      table.foreign_key(columns, options)
+    end
+  end
+end