clickhouse-activerecord 0.3.13 → 0.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 67ea4a63bb140adb5f6de102c38c097ea18f61499a882574952a9982b8d340cf
-  data.tar.gz: a1d2a759dae9062a20cc0a53bbd1cb64142062ce9ef84047f7e953b7d957e9b2
+  metadata.gz: deeb0e0ddf3a6bef04338637ddf814a826f93b4e69779905f28c280a5ca1a400
+  data.tar.gz: 80cba4b321f42cb978be4d6bf4740b03274a7bd0ff882b35d4ac0760736fe9a8
 SHA512:
-  metadata.gz: 243b4d089d6349d8c12c529fbb80078ffbaa03f0033c3c69be77751846b031398895a1efdd08460699636277695a3e03eb228a398b5887a9426f4f3b63496c31
-  data.tar.gz: f790bca5af79492390de98ba2dce363e16207d987093c2e94e9f9ab55b9f769a41bae1c4a12d0b9d3757ee91f92604c1df4db17479dbbc85ced0f48294b297f3
+  metadata.gz: 8c18476f8984ab7e7c834b712eff4f9d8e3995287b3db6ace0e4e819ac19dc6bd31b149fee7da970223807170b1902ce8d2af81ad8ac8a56bc73641e26111d22
+  data.tar.gz: '094cdb750234da2b59efd755c7fef3e4f4bf3484b124d7d2b74fe999ee2a25cddb435324fd88a49248943cb372459df31a3d151905e93118b456231ae94cbd2c'

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+### Version 0.4.4 (Sep 23, 2020)
+
+ * Full support migration and rollback database
+ * Support cluster and replica. Auto inject to SQL queries.
+ * Fix schema dump/load
+ * Can dump schema for using PostgreSQL
+
 ### Version 0.3.10 (Dec 20, 2019)
 
  * Support structure dump/load [@StoneGod](https://github.com/StoneGod)

data/README.md

@@ -1,7 +1,7 @@
 # Clickhouse::Activerecord
 
 A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 5.2.
-Tested on ClickHouse version 18.14.
+Support ClickHouse version from 19.14 LTS.
 
 ## Installation
 
@@ -31,8 +31,8 @@ default: &default
   ssl: true # optional for using ssl connection
   debug: true # use for showing in to log technical information
   migrations_paths: db/clickhouse # optional, default: db/migrate_clickhouse
-  cluster: 'cluster_name' # optional for creating tables in cluster 
-  replica: '{shard}' # optional for creating system tables for shards
+  cluster_name: 'cluster_name' # optional for creating tables in cluster 
+  replica_name: '{replica}' # replica macros name, optional for creating replicated tables
 ```
 
 ## Usage in Rails 5
@@ -111,8 +111,7 @@ Migration:
 
     $ rails g clickhouse_migration MIGRATION_NAME COLUMNS
     $ rake clickhouse:migrate
-    
-Rollback migration not supported!
+    $ rake clickhouse:rollback
 
 ### Dump / Load for multiple using databases
 
@@ -126,7 +125,11 @@ Schema load from `db/clickhouse_schema.rb` file:
 
     $ rake clickhouse:schema:load
 
-We use schema for emulate development or tests environment on PostgreSQL adapter.
+For export schema to PostgreSQL, you need use:
+
+    $ rake clickhouse:schema:dump -- --simple
+
+Schema will be dump to `db/clickhouse_schema_simple.rb`. If default file exists, it will be auto update after migration.
     
 Structure dump to `db/clickhouse_structure.sql` file:
 
@@ -159,6 +162,19 @@ ActionView.maximum(:date)
 #=> 'Wed, 29 Nov 2017'
 ```
 
+### Using replica and cluster params in connection parameters
+
+```yml
+default: &default
+  ***
+  cluster_name: 'cluster_name'
+  replica_name: '{replica}'
+```
+
+`ON CLUSTER cluster_name` will be attach to all queries create / drop.
+
+Engines `MergeTree` and all support replication engines will be replaced to `Replicated***('/clickhouse/tables/cluster_name/database.table', '{replica}')`
+
 ## Donations
 
 Donations to this project are going directly to [PNixx](https://github.com/PNixx), the original author of this project:

data/clickhouse-activerecord.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'activerecord', '>= 5.2'
 
   spec.add_development_dependency 'bundler', '~> 1.15'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.4'
   spec.add_development_dependency 'pry', '~> 0.12'
 end

data/lib/active_record/connection_adapters/clickhouse/schema_creation.rb

@@ -6,15 +6,17 @@ module ActiveRecord
       class SchemaCreation < AbstractAdapter::SchemaCreation# :nodoc:
 
         def visit_AddColumnDefinition(o)
-          +"ADD COLUMN #{accept(o.column)}"
+          sql = +"ADD COLUMN #{accept(o.column)}"
+          sql << " AFTER " + quote_column_name(o.column.options[:after]) if o.column.options.key?(:after)
+          sql
         end
 
         def add_column_options!(sql, options)
-          sql << " DEFAULT #{quote_default_expression(options[:default], options[:column])}" if options_include_default?(options)
           if options[:null] || options[:null].nil?
             sql.gsub!(/\s+(.*)/, ' Nullable(\1)')
           end
           sql.gsub!(/(\sString)\(\d+\)/, '\1')
+          sql << " DEFAULT #{quote_default_expression(options[:default], options[:column])}" if options_include_default?(options)
           sql
         end
 
@@ -27,6 +29,45 @@ module ActiveRecord
 
           create_sql
         end
+
+        def visit_TableDefinition(o)
+          create_sql = +"CREATE#{table_modifier_in_create(o)} #{o.view ? "VIEW" : "TABLE"} "
+          create_sql << "IF NOT EXISTS " if o.if_not_exists
+          create_sql << "#{quote_table_name(o.name)} "
+
+          statements = o.columns.map { |c| accept c }
+          statements << accept(o.primary_keys) if o.primary_keys
+
+          create_sql << "(#{statements.join(', ')})" if statements.present?
+          add_table_options!(create_sql, table_options(o))
+          create_sql << " AS #{to_sql(o.as)}" if o.as
+          create_sql
+        end
+
+        # Returns any SQL string to go between CREATE and TABLE. May be nil.
+        def table_modifier_in_create(o)
+          " TEMPORARY" if o.temporary
+          " MATERIALIZED" if o.materialized
+        end
+
+        def visit_ChangeColumnDefinition(o)
+          column = o.column
+          column.sql_type = type_to_sql(column.type, column.options)
+          options = column_options(column)
+
+          quoted_column_name = quote_column_name(o.name)
+          type = column.sql_type
+          type = "Nullable(#{type})" if options[:null]
+          change_column_sql = +"MODIFY COLUMN #{quoted_column_name} #{type}"
+
+          if options.key?(:default)
+            quoted_default = quote_default_expression(options[:default], column)
+            change_column_sql << " DEFAULT #{quoted_default}"
+          end
+
+          change_column_sql
+        end
+
       end
     end
   end

data/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb

@@ -5,6 +5,35 @@ module ActiveRecord
     module Clickhouse
       class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition
 
+        attr_reader :view, :materialized, :if_not_exists
+
+        def initialize(
+            conn,
+            name,
+            temporary: false,
+            if_not_exists: false,
+            options: nil,
+            as: nil,
+            comment: nil,
+            view: false,
+            materialized: false,
+            **
+          )
+          @conn = conn
+          @columns_hash = {}
+          @indexes = []
+          @foreign_keys = []
+          @primary_keys = nil
+          @temporary = temporary
+          @if_not_exists = if_not_exists
+          @options = options
+          @as = as
+          @name = name
+          @comment = comment
+          @view = view || materialized
+          @materialized = materialized
+        end
+
         def integer(*args, **options)
           if options[:limit] == 8
             args.each { |name| column(name, :big_integer, options.except(:limit)) }

data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb

@@ -67,6 +67,26 @@ module ActiveRecord
           end
         end
 
+        def assume_migrated_upto_version(version, migrations_paths = nil)
+          version = version.to_i
+          sm_table = quote_table_name(schema_migration.table_name)
+
+          migrated = migration_context.get_all_versions
+          versions = migration_context.migrations.map(&:version)
+
+          unless migrated.include?(version)
+            exec_insert "INSERT INTO #{sm_table} (version) VALUES (#{quote(version.to_s)})", nil, nil
+          end
+
+          inserting = (versions - migrated).select { |v| v < version }
+          if inserting.any?
+            if (duplicate = inserting.detect { |v| inserting.count(v) > 1 })
+              raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict."
+            end
+            execute insert_versions_sql(inserting)
+          end
+        end
+
         private
 
         def apply_format(sql, format)
@@ -95,11 +115,7 @@ module ActiveRecord
         end
 
         def create_table_definition(*args)
-          if ActiveRecord::version >= Gem::Version.new('6')
-            Clickhouse::TableDefinition.new(self, *args)
-          else
-            Clickhouse::TableDefinition.new(*args)
-          end
+          Clickhouse::TableDefinition.new(self, *args)
         end
 
         def new_column_from_field(table_name, field)
@@ -140,6 +156,8 @@ module ActiveRecord
           when "true".freeze, "false".freeze
             default
             # Object identifier types
+          when "''"
+            ''
           when /\A-?\d+\z/
             $1
           else

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -2,6 +2,7 @@
 
 require 'clickhouse-activerecord/arel/visitors/to_sql'
 require 'clickhouse-activerecord/arel/table'
+require 'clickhouse-activerecord/migration'
 require 'active_record/connection_adapters/clickhouse/oid/date'
 require 'active_record/connection_adapters/clickhouse/oid/date_time'
 require 'active_record/connection_adapters/clickhouse/oid/big_integer'
@@ -110,17 +111,17 @@ module ActiveRecord
 
       # Support SchemaMigration from v5.2.2 to v6+
       def schema_migration # :nodoc:
-        if ActiveRecord::version >= Gem::Version.new('6')
-          super
-        else
-          ActiveRecord::SchemaMigration
-        end
+        ClickhouseActiverecord::SchemaMigration
       end
 
       def migrations_paths
         @full_config[:migrations_paths] || 'db/migrate_clickhouse'
       end
 
+      def migration_context # :nodoc:
+        ClickhouseActiverecord::MigrationContext.new(migrations_paths, schema_migration)
+      end
+
       def arel_visitor # :nodoc:
         ClickhouseActiverecord::Arel::Visitors::ToSql.new(self)
       end
@@ -204,17 +205,34 @@ module ActiveRecord
       def create_database(name)
         sql = apply_cluster "CREATE DATABASE #{quote_table_name(name)}"
         log_with_debug(sql, adapter_name) do
-          res = @connection.post("/?#{@config.except(:database).to_param}", "CREATE DATABASE #{quote_table_name(name)}")
+          res = @connection.post("/?#{@config.except(:database).to_param}", sql)
           process_response(res)
         end
       end
 
-      def create_table(table_name, comment: nil, **options)
-        super(
-          apply_cluster(table_name),
-          comment: comment,
-          **options
-        )
+      def create_view(table_name, **options)
+        options.merge!(view: true)
+        options = apply_replica(table_name, options)
+        td = create_table_definition(apply_cluster(table_name), options)
+        yield td if block_given?
+
+        if options[:force]
+          drop_table(table_name, options.merge(if_exists: true))
+        end
+
+        execute schema_creation.accept td
+      end
+
+      def create_table(table_name, **options)
+        options = apply_replica(table_name, options)
+        td = create_table_definition(apply_cluster(table_name), options)
+        yield td if block_given?
+
+        if options[:force]
+          drop_table(table_name, options.merge(if_exists: true))
+        end
+
+        execute schema_creation.accept td
       end
 
       # Drops a ClickHouse database.
@@ -226,28 +244,71 @@ module ActiveRecord
         end
       end
 
+      def rename_table(table_name, new_name)
+        do_execute apply_cluster "RENAME TABLE #{quote_table_name(table_name)} TO #{quote_table_name(new_name)}"
+      end
+
       def drop_table(table_name, options = {}) # :nodoc:
         do_execute apply_cluster "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}"
       end
 
+      def change_column(table_name, column_name, type, options = {})
+        result = do_execute "ALTER TABLE #{quote_table_name(table_name)} #{change_column_for_alter(table_name, column_name, type, options)}"
+        raise "Error parse json response: #{result}" if result.presence && !result.is_a?(Hash)
+      end
+
+      def change_column_null(table_name, column_name, null, default = nil)
+        structure = table_structure(table_name).select{|v| v[0] == column_name.to_s}.first
+        raise "Column #{column_name} not found in table #{table_name}" if structure.nil?
+        change_column table_name, column_name, structure[1].gsub(/(Nullable\()?(.*?)\)?/, '\2'), {null: null, default: default}.compact
+      end
+
+      def change_column_default(table_name, column_name, default)
+        change_column table_name, column_name, nil, {default: default}.compact
+      end
+
+      def cluster
+        @full_config[:cluster_name]
+      end
+
+      def replica
+        @full_config[:replica_name]
+      end
+
+      def replica_path(table)
+        "/clickhouse/tables/#{cluster}/#{@config[:database]}.#{table}"
+      end
+
+      def apply_cluster(sql)
+        cluster ? "#{sql} ON CLUSTER #{cluster}" : sql
+      end
+
       protected
 
       def last_inserted_id(result)
         result
       end
 
+      def change_column_for_alter(table_name, column_name, type, options = {})
+        td = create_table_definition(table_name)
+        cd = td.new_column_definition(column_name, type, options)
+        schema_creation.accept(ChangeColumnDefinition.new(cd, column_name))
+      end
+
       private
 
       def connect
         @connection = Net::HTTP.start(@connection_parameters[0], @connection_parameters[1], use_ssl: @connection_parameters[2], verify_mode: OpenSSL::SSL::VERIFY_NONE)
       end
 
-      def cluster
-        @full_config[:cluster]
-      end
-
-      def apply_cluster(sql)
-        cluster ? "#{sql} ON CLUSTER #{cluster}" : sql
+      def apply_replica(table, options)
+        if replica && cluster && options[:options]
+          match = options[:options].match(/^(.*?MergeTree)\(([^\)]*)\)(.*?)$/)
+          if match
+            options[:options] = "Replicated#{match[1]}(#{([replica_path(table), replica].map{|v| "'#{v}'"} + [match[2].presence]).compact.join(', ')})#{match[3]}"
+          end
+        end
+        options
       end
     end
   end

data/lib/clickhouse-activerecord.rb

@@ -4,6 +4,7 @@ require 'active_record/connection_adapters/clickhouse_adapter'
 
 if defined?(Rails::Railtie)
   require 'clickhouse-activerecord/railtie'
+  require 'clickhouse-activerecord/schema'
   require 'clickhouse-activerecord/schema_dumper'
   require 'clickhouse-activerecord/tasks'
   ActiveRecord::Tasks::DatabaseTasks.register_task(/clickhouse/, "ClickhouseActiverecord::Tasks")

data/lib/clickhouse-activerecord/migration.rb

@@ -0,0 +1,92 @@
+module ClickhouseActiverecord
+
+  class SchemaMigration < ::ActiveRecord::SchemaMigration
+    class << self
+
+      def create_table
+        unless table_exists?
+          version_options = connection.internal_string_options_for_primary_key
+
+          connection.create_table(table_name, id: false, options: 'ReplacingMergeTree(ver) PARTITION BY version ORDER BY (version)') do |t|
+            t.string :version, version_options
+            t.column :active, 'Int8', null: false, default: '1'
+            t.datetime :ver, null: false, default: -> { 'now()' }
+          end
+        end
+      end
+
+      def all_versions
+        from("#{table_name} FINAL").where(active: 1).order(:version).pluck(:version)
+      end
+    end
+  end
+
+  class InternalMetadata < ::ActiveRecord::InternalMetadata
+    class << self
+      def create_table
+        unless table_exists?
+          key_options = connection.internal_string_options_for_primary_key
+
+          connection.create_table(table_name, id: false, options: 'MergeTree() PARTITION BY toDate(created_at) ORDER BY (created_at)') do |t|
+            t.string :key, key_options
+            t.string :value
+            t.timestamps
+          end
+        end
+      end
+    end
+  end
+
+  class MigrationContext < ::ActiveRecord::MigrationContext #:nodoc:
+    attr_reader :migrations_paths, :schema_migration
+
+    def initialize(migrations_paths, schema_migration)
+      @migrations_paths = migrations_paths
+      @schema_migration = schema_migration
+    end
+
+    def down(target_version = nil)
+      selected_migrations = if block_given?
+        migrations.select { |m| yield m }
+      else
+        migrations
+      end
+
+      ClickhouseActiverecord::Migrator.new(:down, selected_migrations, schema_migration, target_version).migrate
+    end
+
+    def get_all_versions
+      if schema_migration.table_exists?
+        schema_migration.all_versions.map(&:to_i)
+      else
+        []
+      end
+    end
+
+  end
+
+  class Migrator < ::ActiveRecord::Migrator
+
+    def initialize(direction, migrations, schema_migration, target_version = nil)
+      @direction         = direction
+      @target_version    = target_version
+      @migrated_versions = nil
+      @migrations        = migrations
+      @schema_migration  = schema_migration
+
+      validate(@migrations)
+
+      @schema_migration.create_table
+      ClickhouseActiverecord::InternalMetadata.create_table
+    end
+
+    def record_version_state_after_migrating(version)
+      if down?
+        migrated.delete(version)
+        @schema_migration.create!(version: version.to_s, active: 0)
+      else
+        super
+      end
+    end
+  end
+end

data/lib/clickhouse-activerecord/schema.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module ClickhouseActiverecord
+
+  class Schema < ::ActiveRecord::Schema
+
+    def define(info, &block) # :nodoc:
+      instance_eval(&block)
+
+      if info[:version].present?
+        connection.schema_migration.create_table
+        connection.assume_migrated_upto_version(info[:version], ClickhouseActiverecord::Migrator.migrations_paths)
+      end
+
+      ClickhouseActiverecord::InternalMetadata.create_table
+      ClickhouseActiverecord::InternalMetadata[:environment] = connection.migration_context.current_environment
+    end
+  end
+end

data/lib/clickhouse-activerecord/schema_dumper.rb

@@ -1,10 +1,129 @@
 module ClickhouseActiverecord
   class SchemaDumper < ::ActiveRecord::ConnectionAdapters::SchemaDumper
 
+    attr_accessor :simple
+
+    class << self
+      def dump(connection = ActiveRecord::Base.connection, stream = STDOUT, config = ActiveRecord::Base, default = false)
+        dumper = connection.create_schema_dumper(generate_options(config))
+        dumper.simple = default
+        dumper.dump(stream)
+        stream
+      end
+    end
+
+    private
+
+    def header(stream)
+      stream.puts <<HEADER
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# This file is the source Rails uses to define your schema when running `rails
+# #{simple ? 'db' : 'clickhouse'}:schema:load`. When creating a new database, `rails #{simple ? 'db' : 'clickhouse'}:schema:load` tends to
+# be faster and is potentially less error prone than running all of your
+# migrations from scratch. Old migrations may fail to apply correctly if those
+# migrations use external dependencies or application code.
+#
+# It's strongly recommended that you check this file into your version control system.
+
+#{simple ? 'ActiveRecord' : 'ClickhouseActiverecord'}::Schema.define(#{define_params}) do
+
+HEADER
+    end
+
     def table(table, stream)
-      stream.puts "  # TABLE: #{table}"
-      stream.puts "  # SQL: #{@connection.do_system_execute("SHOW CREATE TABLE `#{table.gsub(/^\.inner\./, '')}`")['data'].try(:first).try(:first)}"
-      super(table.gsub(/^\.inner\./, ''), stream)
+      if table.match(/^\.inner\./).nil?
+        unless simple
+          stream.puts "  # TABLE: #{table}"
+          sql = @connection.do_system_execute("SHOW CREATE TABLE `#{table.gsub(/^\.inner\./, '')}`")['data'].try(:first).try(:first)
+          stream.puts "  # SQL: #{sql.gsub(/ENGINE = Replicated(.*?)\('[^']+',\s*'[^']+',?\s?([^\)]*)?\)/, "ENGINE = \\1(\\2)")}" if sql
+          # super(table.gsub(/^\.inner\./, ''), stream)
+
+          # detect view table
+          match = sql.match(/^CREATE\s+(MATERIALIZED)\s+VIEW/)
+        end
+
+        # Copy from original dumper
+        columns = @connection.columns(table)
+        begin
+          tbl = StringIO.new
+
+          # first dump primary key column
+          pk = @connection.primary_key(table)
+
+          tbl.print "  create_table #{remove_prefix_and_suffix(table).inspect}"
+
+          unless simple
+            # Add materialize flag
+            tbl.print ', view: true' if match
+            tbl.print ', materialized: true' if match && match[1].presence
+          end
+
+          case pk
+          when String
+            tbl.print ", primary_key: #{pk.inspect}" unless pk == "id"
+            pkcol = columns.detect { |c| c.name == pk }
+            pkcolspec = column_spec_for_primary_key(pkcol)
+            if pkcolspec.present?
+              tbl.print ", #{format_colspec(pkcolspec)}"
+            end
+          when Array
+            tbl.print ", primary_key: #{pk.inspect}"
+          else
+            tbl.print ", id: false"
+          end
+
+          unless simple
+            table_options = @connection.table_options(table)
+            if table_options.present?
+              tbl.print ", #{format_options(table_options)}"
+            end
+          end
+
+          tbl.puts ", force: :cascade do |t|"
+
+          # then dump all non-primary key columns
+          if simple || !match
+            columns.each do |column|
+              raise StandardError, "Unknown type '#{column.sql_type}' for column '#{column.name}'" unless @connection.valid_type?(column.type)
+              next if column.name == pk
+              type, colspec = column_spec(column)
+              tbl.print "    t.#{type} #{column.name.inspect}"
+              tbl.print ", #{format_colspec(colspec)}" if colspec.present?
+              tbl.puts
+            end
+          end
+
+          indexes_in_create(table, tbl)
+
+          tbl.puts "  end"
+          tbl.puts
+
+          tbl.rewind
+          stream.print tbl.read
+        rescue => e
+          stream.puts "# Could not dump table #{table.inspect} because of following #{e.class}"
+          stream.puts "#   #{e.message}"
+          stream.puts
+        end
+      end
+    end
+
+    def format_options(options)
+      if options && options[:options]
+        options[:options] = options[:options].gsub(/^Replicated(.*?)\('[^']+',\s*'[^']+',?\s?([^\)]*)?\)/, "\\1(\\2)")
+      end
+      super
+    end
+
+    def format_colspec(colspec)
+      if simple
+        super.gsub(/CAST\('?([^,']*)'?,\s?'.*?'\)/, "\\1")
+      else
+        super
+      end
     end
   end
 end

data/lib/clickhouse-activerecord/version.rb

@@ -1,3 +1,3 @@
 module ClickhouseActiverecord
-  VERSION = '0.3.13'
+  VERSION = '0.4.6'
 end

data/lib/generators/clickhouse_migration_generator.rb

@@ -12,8 +12,8 @@ class ClickhouseMigrationGenerator < ActiveRecord::Generators::MigrationGenerato
   private
 
   def db_migrate_path
-    if defined?(Rails.application) && Rails.application
-      configured_migrate_path || default_migrate_path
+    if defined?(Rails.application) && Rails.application && respond_to?(:configured_migrate_path, true)
+      configured_migrate_path
     else
       default_migrate_path
     end

data/lib/tasks/clickhouse.rake

@@ -3,67 +3,11 @@
 namespace :clickhouse do
 
   task prepare_schema_migration_table: :environment do
-    cluster, database, replica = ActiveRecord::Base.connection_config.values_at(:cluster, :database, :replica)
-    return if cluster.nil?
-
-    connection = ActiveRecord::Base.connection
-    key_options = connection.internal_string_options_for_primary_key
-    block = Proc.new do |t|
-      t.string :version, key_options
-    end
-    distributed_table_name = ".#{ActiveRecord::SchemaMigration.table_name}_distributed"
-    unless connection.table_exists?(distributed_table_name)
-      options = { id: false }
-      if replica
-        shard = replica.is_a?(String) ? replica : '{shard}'
-        options[:options] = <<-SQL
-          ReplicatedMergeTree('/clickhouse/tables/{cluster}/#{shard}/#{database}.`#{distributed_table_name}`', '{replica}')
-          PARTITION BY version ORDER BY (version) SETTINGS index_granularity = 8192
-        SQL
-      end
-      connection.create_table("`#{distributed_table_name}`", options, &block)
-    end
-    unless connection.table_exists?(ActiveRecord::SchemaMigration.table_name)
-      connection.create_table(
-        ActiveRecord::SchemaMigration.table_name,
-        id: false,
-        options: "Distributed(#{cluster},#{database},`#{distributed_table_name}`,sipHash64(version))",
-        &block
-      )
-    end
+    ClickhouseActiverecord::SchemaMigration.create_table unless ENV['simple'] || ARGV.map{|a| a.include?('--simple') ? true : nil}.compact.any?
   end
 
   task prepare_internal_metadata_table: :environment do
-    cluster, database, replica = ActiveRecord::Base.connection_config.values_at(:cluster, :database, :replica)
-    return if cluster.nil?
-
-    connection = ActiveRecord::Base.connection
-    key_options = connection.internal_string_options_for_primary_key
-    block = Proc.new do |t|
-      t.string :key, key_options
-      t.string :value
-      t.timestamps
-    end
-    distributed_table_name = ".#{ActiveRecord::InternalMetadata.table_name}_distributed"
-    unless connection.table_exists?(distributed_table_name)
-      options = { id: false }
-      if replica
-        shard = replica.is_a?(String) ? replica : '{shard}'
-        options[:options] = <<-SQL
-          ReplicatedMergeTree('/clickhouse/tables/{cluster}/#{shard}/#{database}.`#{distributed_table_name}`', '{replica}')
-          PARTITION BY toDate(created_at) ORDER BY (created_at) SETTINGS index_granularity = 8192
-        SQL
-      end
-      connection.create_table("`#{distributed_table_name}`", options, &block)
-    end
-    unless connection.table_exists?(ActiveRecord::InternalMetadata.table_name)
-      connection.create_table(
-        ActiveRecord::InternalMetadata.table_name,
-        id: false,
-        options: "Distributed(#{cluster},#{database},`#{distributed_table_name}`,sipHash64(created_at))",
-        &block
-      )
-    end
+    ClickhouseActiverecord::InternalMetadata.create_table unless ENV['simple'] || ARGV.map{|a| a.include?('--simple') ? true : nil}.compact.any?
   end
 
   task load_config: :environment do
@@ -76,16 +20,19 @@ namespace :clickhouse do
 
     # todo not testing
     desc 'Load database schema'
-    task load: :load_config do
-      load("#{Rails.root}/db/clickhouse_schema.rb")
+    task load: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do |t, args|
+      simple = ENV['simple'] || ARGV.map{|a| a.include?('--simple') ? true : nil}.compact.any? ? '_simple' : nil
+      ClickhouseActiverecord::SchemaMigration.drop_table
+      load("#{Rails.root}/db/clickhouse_schema#{simple}.rb")
     end
 
     desc 'Dump database schema'
-    task dump: :environment do
-      filename = "#{Rails.root}/db/clickhouse_schema.rb"
+    task dump: :environment do |t, args|
+      simple = ENV['simple'] || args[:simple] || ARGV.map{|a| a.include?('--simple') ? true : nil}.compact.any? ? '_simple' : nil
+      filename = "#{Rails.root}/db/clickhouse_schema#{simple}.rb"
       File.open(filename, 'w:utf-8') do |file|
         ActiveRecord::Base.establish_connection(:"#{Rails.env}_clickhouse")
-        ClickhouseActiverecord::SchemaDumper.dump(ActiveRecord::Base.connection, file)
+        ClickhouseActiverecord::SchemaDumper.dump(ActiveRecord::Base.connection, file, ActiveRecord::Base, !!simple)
       end
     end
 
@@ -127,5 +74,13 @@ namespace :clickhouse do
   desc 'Migrate the clickhouse database'
   task migrate: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do
     Rake::Task['db:migrate'].execute
+    if File.exists? "#{Rails.root}/db/clickhouse_schema_simple.rb"
+      Rake::Task['clickhouse:schema:dump'].execute(simple: true)
+    end
+  end
+
+  desc 'Rollback the clickhouse database'
+  task rollback: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do
+    Rake::Task['db:rollback'].execute
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clickhouse-activerecord
 version: !ruby/object:Gem::Version
-  version: 0.3.13
+  version: 0.4.6
 platform: ruby
 authors:
 - Sergey Odintsov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-11 00:00:00.000000000 Z
+date: 2020-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -122,7 +122,9 @@ files:
 - lib/clickhouse-activerecord.rb
 - lib/clickhouse-activerecord/arel/table.rb
 - lib/clickhouse-activerecord/arel/visitors/to_sql.rb
+- lib/clickhouse-activerecord/migration.rb
 - lib/clickhouse-activerecord/railtie.rb
+- lib/clickhouse-activerecord/schema.rb
 - lib/clickhouse-activerecord/schema_dumper.rb
 - lib/clickhouse-activerecord/tasks.rb
 - lib/clickhouse-activerecord/version.rb