clickhouse-activerecord 0.3.12 → 0.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 50dff8ae7219e2f3a04ee530331d19ca0b61396437723819891c1d8b41f0692d
-  data.tar.gz: 75d50e8e2f32fa84f990bc15da6177082b7a0b0d5a508cba822b3c3afbc60c41
+  metadata.gz: 44a359e720da3f43b923698d6df95231f003768e160fe625ace18ec442469f50
+  data.tar.gz: edd7a9cebc8e1b1f16a1e5a44bafdde4bca4697d88336c7b00961109f6062756
 SHA512:
-  metadata.gz: ebdeb04ba96eb81eb76efdbd09580b51719656136de38b973cea267bc1405dba03cd417058df1909b9bd1e4279c97d845292947de251792f94fc9f0b30cf65cb
-  data.tar.gz: 3a0b0feda885ab74f54f6d61b78ff02efed20d403b0bc4bff99b83ee811d4988e8620c92a84e549bc70e0ccb93c662e665a562bb51d49b5ed552bc850c076696
+  metadata.gz: 9c049f566c462602f61c883aa1fb11a2fef92806ae687e2550be21a995dcb0f6b4b0dc9cd197b30aa2918ef72b9c9db892d0eb62db8f5861db1e8c5eb498d114
+  data.tar.gz: 83dbd6124dcd48b85bd941510f747f08ebea868d65bbecfc6133a52d8fe40b2392906ee883179d3b5f644e2531801b225f56d9bd31377391b905fcf490b327f0

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+### Version 0.4.4 (Sep 23, 2020)
+
+ * Full support migration and rollback database
+ * Support cluster and replica. Auto inject to SQL queries.
+ * Fix schema dump/load
+ * Can dump schema for using PostgreSQL
+
 ### Version 0.3.10 (Dec 20, 2019)
 
  * Support structure dump/load [@StoneGod](https://github.com/StoneGod)

data/README.md

@@ -1,7 +1,7 @@
 # Clickhouse::Activerecord
 
 A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 5.2.
-Tested on ClickHouse version 18.14.
+Support ClickHouse version from 19.14 LTS.
 
 ## Installation
 
@@ -31,6 +31,8 @@ default: &default
   ssl: true # optional for using ssl connection
   debug: true # use for showing in to log technical information
   migrations_paths: db/clickhouse # optional, default: db/migrate_clickhouse
+  cluster_name: 'cluster_name' # optional for creating tables in cluster 
+  replica_name: '{replica}' # replica macros name, optional for creating replicated tables
 ```
 
 ## Usage in Rails 5
@@ -99,13 +101,17 @@ Create / drop / purge / reset database:
     $ rake clickhouse:drop
     $ rake clickhouse:purge
     $ rake clickhouse:reset
+
+Prepare system tables for rails:
+
+    $ rake clickhouse:prepare_schema_migration_table
+    $ rake clickhouse:prepare_internal_metadata_table
     
 Migration:
 
     $ rails g clickhouse_migration MIGRATION_NAME COLUMNS
     $ rake clickhouse:migrate
-    
-Rollback migration not supported!
+    $ rake clickhouse:rollback
 
 ### Dump / Load for multiple using databases
 
@@ -119,7 +125,11 @@ Schema load from `db/clickhouse_schema.rb` file:
 
     $ rake clickhouse:schema:load
 
-We use schema for emulate development or tests environment on PostgreSQL adapter.
+For export schema to PostgreSQL, you need use:
+
+    $ rake clickhouse:schema:dump -- --simple
+
+Schema will be dump to `db/clickhouse_schema_simple.rb`. If default file exists, it will be auto update after migration.
     
 Structure dump to `db/clickhouse_structure.sql` file:
 
@@ -152,6 +162,19 @@ ActionView.maximum(:date)
 #=> 'Wed, 29 Nov 2017'
 ```
 
+### Using replica and cluster params in connection parameters
+
+```yml
+default: &default
+  ***
+  cluster_name: 'cluster_name'
+  replica_name: '{replica}'
+```
+
+`ON CLUSTER cluster_name` will be attach to all queries create / drop.
+
+Engines `MergeTree` and all support replication engines will be replaced to `Replicated***('/clickhouse/tables/cluster_name/database.table', '{replica}')`
+
 ## Donations
 
 Donations to this project are going directly to [PNixx](https://github.com/PNixx), the original author of this project:

data/clickhouse-activerecord.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'activerecord', '>= 5.2'
 
   spec.add_development_dependency 'bundler', '~> 1.15'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.4'
   spec.add_development_dependency 'pry', '~> 0.12'
 end

data/lib/active_record/connection_adapters/clickhouse/schema_creation.rb

@@ -6,15 +6,17 @@ module ActiveRecord
       class SchemaCreation < AbstractAdapter::SchemaCreation# :nodoc:
 
         def visit_AddColumnDefinition(o)
-          +"ADD COLUMN #{accept(o.column)}"
+          sql = +"ADD COLUMN #{accept(o.column)}"
+          sql << " AFTER " + quote_column_name(o.column.options[:after]) if o.column.options.key?(:after)
+          sql
         end
 
         def add_column_options!(sql, options)
-          sql << " DEFAULT #{quote_default_expression(options[:default], options[:column])}" if options_include_default?(options)
           if options[:null] || options[:null].nil?
             sql.gsub!(/\s+(.*)/, ' Nullable(\1)')
           end
           sql.gsub!(/(\sString)\(\d+\)/, '\1')
+          sql << " DEFAULT #{quote_default_expression(options[:default], options[:column])}" if options_include_default?(options)
           sql
         end
 
@@ -27,6 +29,45 @@ module ActiveRecord
 
           create_sql
         end
+
+        def visit_TableDefinition(o)
+          create_sql = +"CREATE#{table_modifier_in_create(o)} #{o.view ? "VIEW" : "TABLE"} "
+          create_sql << "IF NOT EXISTS " if o.if_not_exists
+          create_sql << "#{quote_table_name(o.name)} "
+
+          statements = o.columns.map { |c| accept c }
+          statements << accept(o.primary_keys) if o.primary_keys
+
+          create_sql << "(#{statements.join(', ')})" if statements.present?
+          add_table_options!(create_sql, table_options(o))
+          create_sql << " AS #{to_sql(o.as)}" if o.as
+          create_sql
+        end
+
+        # Returns any SQL string to go between CREATE and TABLE. May be nil.
+        def table_modifier_in_create(o)
+          " TEMPORARY" if o.temporary
+          " MATERIALIZED" if o.materialized
+        end
+
+        def visit_ChangeColumnDefinition(o)
+          column = o.column
+          column.sql_type = type_to_sql(column.type, column.options)
+          options = column_options(column)
+
+          quoted_column_name = quote_column_name(o.name)
+          type = column.sql_type
+          type = "Nullable(#{type})" if options[:null]
+          change_column_sql = +"MODIFY COLUMN #{quoted_column_name} #{type}"
+
+          if options.key?(:default)
+            quoted_default = quote_default_expression(options[:default], column)
+            change_column_sql << " DEFAULT #{quoted_default}"
+          end
+
+          change_column_sql
+        end
+
       end
     end
   end

data/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb

@@ -5,6 +5,35 @@ module ActiveRecord
     module Clickhouse
       class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition
 
+        attr_reader :view, :materialized, :if_not_exists
+
+        def initialize(
+            conn,
+            name,
+            temporary: false,
+            if_not_exists: false,
+            options: nil,
+            as: nil,
+            comment: nil,
+            view: false,
+            materialized: false,
+            **
+          )
+          @conn = conn
+          @columns_hash = {}
+          @indexes = []
+          @foreign_keys = []
+          @primary_keys = nil
+          @temporary = temporary
+          @if_not_exists = if_not_exists
+          @options = options
+          @as = as
+          @name = name
+          @comment = comment
+          @view = view || materialized
+          @materialized = materialized
+        end
+
         def integer(*args, **options)
           if options[:limit] == 8
             args.each { |name| column(name, :big_integer, options.except(:limit)) }

data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb

@@ -17,7 +17,7 @@ module ActiveRecord
         def exec_query(sql, name = nil, binds = [], prepare: false)
           result = do_execute(sql, name)
           ActiveRecord::Result.new(result['meta'].map { |m| m['name'] }, result['data'])
-        rescue StandartError => _e
+        rescue StandardError => _e
           raise ActiveRecord::ActiveRecordError, "Response: #{result}"
         end
 
@@ -36,7 +36,7 @@ module ActiveRecord
         end
 
         def table_options(table)
-          sql = do_system_execute("SHOW CREATE TABLE #{table}")['data'].try(:first).try(:first)
+          sql = do_system_execute("SHOW CREATE TABLE `#{table}`")['data'].try(:first).try(:first)
           { options: sql.gsub(/^(?:.*?)ENGINE = (.*?)$/, '\\1') }
         end
 
@@ -67,6 +67,26 @@ module ActiveRecord
           end
         end
 
+        def assume_migrated_upto_version(version, migrations_paths = nil)
+          version = version.to_i
+          sm_table = quote_table_name(schema_migration.table_name)
+
+          migrated = migration_context.get_all_versions
+          versions = migration_context.migrations.map(&:version)
+
+          unless migrated.include?(version)
+            exec_insert "INSERT INTO #{sm_table} (version) VALUES (#{quote(version.to_s)})", nil, nil
+          end
+
+          inserting = (versions - migrated).select { |v| v < version }
+          if inserting.any?
+            if (duplicate = inserting.detect { |v| inserting.count(v) > 1 })
+              raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict."
+            end
+            execute insert_versions_sql(inserting)
+          end
+        end
+
         private
 
         def apply_format(sql, format)
@@ -95,11 +115,7 @@ module ActiveRecord
         end
 
         def create_table_definition(*args)
-          if ActiveRecord::version >= Gem::Version.new('6')
-            Clickhouse::TableDefinition.new(self, *args)
-          else
-            Clickhouse::TableDefinition.new(*args)
-          end
+          Clickhouse::TableDefinition.new(self, *args)
         end
 
         def new_column_from_field(table_name, field)
@@ -118,7 +134,7 @@ module ActiveRecord
         protected
 
         def table_structure(table_name)
-          result = do_system_execute("DESCRIBE TABLE #{table_name}", table_name)
+          result = do_system_execute("DESCRIBE TABLE `#{table_name}`", table_name)
           data = result['data']
 
           return data unless data.empty?
@@ -140,6 +156,8 @@ module ActiveRecord
           when "true".freeze, "false".freeze
             default
             # Object identifier types
+          when "''"
+            ''
           when /\A-?\d+\z/
             $1
           else

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -2,6 +2,7 @@
 
 require 'clickhouse-activerecord/arel/visitors/to_sql'
 require 'clickhouse-activerecord/arel/table'
+require 'clickhouse-activerecord/migration'
 require 'active_record/connection_adapters/clickhouse/oid/date'
 require 'active_record/connection_adapters/clickhouse/oid/date_time'
 require 'active_record/connection_adapters/clickhouse/oid/big_integer'
@@ -110,17 +111,17 @@ module ActiveRecord
 
       # Support SchemaMigration from v5.2.2 to v6+
       def schema_migration # :nodoc:
-        if ActiveRecord::version >= Gem::Version.new('6')
-          super
-        else
-          ActiveRecord::SchemaMigration
-        end
+        ClickhouseActiverecord::SchemaMigration
       end
 
       def migrations_paths
         @full_config[:migrations_paths] || 'db/migrate_clickhouse'
       end
 
+      def migration_context # :nodoc:
+        ClickhouseActiverecord::MigrationContext.new(migrations_paths, schema_migration)
+      end
+
       def arel_visitor # :nodoc:
         ClickhouseActiverecord::Arel::Visitors::ToSql.new(self)
       end
@@ -202,24 +203,84 @@ module ActiveRecord
 
       # Create a new ClickHouse database.
       def create_database(name)
-        sql = "CREATE DATABASE #{quote_table_name(name)}"
+        sql = apply_cluster "CREATE DATABASE #{quote_table_name(name)}"
         log_with_debug(sql, adapter_name) do
-          res = @connection.post("/?#{@config.except(:database).to_param}", "CREATE DATABASE #{quote_table_name(name)}")
+          res = @connection.post("/?#{@config.except(:database).to_param}", sql)
           process_response(res)
         end
       end
 
+      def create_view(table_name, **options)
+        options.merge!(view: true)
+        options = apply_replica(table_name, options)
+        td = create_table_definition(apply_cluster(table_name), options)
+        yield td if block_given?
+
+        if options[:force]
+          drop_table(table_name, options.merge(if_exists: true))
+        end
+
+        execute schema_creation.accept td
+      end
+
+      def create_table(table_name, **options)
+        options = apply_replica(table_name, options)
+        td = create_table_definition(apply_cluster(table_name), options)
+        yield td if block_given?
+
+        if options[:force]
+          drop_table(table_name, options.merge(if_exists: true))
+        end
+
+        execute schema_creation.accept td
+      end
+
       # Drops a ClickHouse database.
       def drop_database(name) #:nodoc:
-        sql = "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
+        sql = apply_cluster "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
         log_with_debug(sql, adapter_name) do
           res = @connection.post("/?#{@config.except(:database).to_param}", sql)
           process_response(res)
         end
       end
 
+      def rename_table(table_name, new_name)
+        do_execute apply_cluster "RENAME TABLE #{quote_table_name(table_name)} TO #{quote_table_name(new_name)}"
+      end
+
       def drop_table(table_name, options = {}) # :nodoc:
-        do_execute "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}"
+        do_execute apply_cluster "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}"
+      end
+
+      def change_column(table_name, column_name, type, options = {})
+        result = do_execute "ALTER TABLE #{quote_table_name(table_name)} #{change_column_for_alter(table_name, column_name, type, options)}"
+        raise "Error parse json response: #{result}" if result.presence && !result.is_a?(Hash)
+      end
+
+      def change_column_null(table_name, column_name, null, default = nil)
+        structure = table_structure(table_name).select{|v| v[0] == column_name.to_s}.first
+        raise "Column #{column_name} not found in table #{table_name}" if structure.nil?
+        change_column table_name, column_name, structure[1].gsub(/(Nullable\()?(.*?)\)?/, '\2'), {null: null, default: default}.compact
+      end
+
+      def change_column_default(table_name, column_name, default)
+        change_column table_name, column_name, nil, {default: default}.compact
+      end
+
+      def cluster
+        @full_config[:cluster_name]
+      end
+
+      def replica
+        @full_config[:replica_name]
+      end
+
+      def replica_path(table)
+        "/clickhouse/tables/#{cluster}/#{@config[:database]}.#{table}"
+      end
+
+      def apply_cluster(sql)
+        cluster ? "#{sql} ON CLUSTER #{cluster}" : sql
       end
 
       protected
@@ -228,11 +289,27 @@ module ActiveRecord
         result
       end
 
+      def change_column_for_alter(table_name, column_name, type, options = {})
+        td = create_table_definition(table_name)
+        cd = td.new_column_definition(column_name, type, options)
+        schema_creation.accept(ChangeColumnDefinition.new(cd, column_name))
+      end
+
       private
 
       def connect
         @connection = Net::HTTP.start(@connection_parameters[0], @connection_parameters[1], use_ssl: @connection_parameters[2], verify_mode: OpenSSL::SSL::VERIFY_NONE)
       end
+
+      def apply_replica(table, options)
+        if replica && cluster && options[:options]
+          match = options[:options].match(/^(.*?MergeTree)\(([^\)]*)\)(.*?)$/)
+          if match
+            options[:options] = "Replicated#{match[1]}(#{([replica_path(table), replica].map{|v| "'#{v}'"} + [match[2].presence]).compact.join(', ')})#{match[3]}"
+          end
+        end
+        options
+      end
     end
   end
 end

data/lib/clickhouse-activerecord.rb

@@ -4,6 +4,7 @@ require 'active_record/connection_adapters/clickhouse_adapter'
 
 if defined?(Rails::Railtie)
   require 'clickhouse-activerecord/railtie'
+  require 'clickhouse-activerecord/schema'
   require 'clickhouse-activerecord/schema_dumper'
   require 'clickhouse-activerecord/tasks'
   ActiveRecord::Tasks::DatabaseTasks.register_task(/clickhouse/, "ClickhouseActiverecord::Tasks")

data/lib/clickhouse-activerecord/migration.rb

@@ -0,0 +1,92 @@
+module ClickhouseActiverecord
+
+  class SchemaMigration < ::ActiveRecord::SchemaMigration
+    class << self
+
+      def create_table
+        unless table_exists?
+          version_options = connection.internal_string_options_for_primary_key
+
+          connection.create_table(table_name, id: false, options: 'ReplacingMergeTree(ver) PARTITION BY version ORDER BY (version)') do |t|
+            t.string :version, version_options
+            t.column :active, 'Int8', null: false, default: '1'
+            t.datetime :ver, null: false, default: -> { 'now()' }
+          end
+        end
+      end
+
+      def all_versions
+        from("#{table_name} FINAL").where(active: 1).order(:version).pluck(:version)
+      end
+    end
+  end
+
+  class InternalMetadata < ::ActiveRecord::InternalMetadata
+    class << self
+      def create_table
+        unless table_exists?
+          key_options = connection.internal_string_options_for_primary_key
+
+          connection.create_table(table_name, id: false, options: 'MergeTree() PARTITION BY toDate(created_at) ORDER BY (created_at)') do |t|
+            t.string :key, key_options
+            t.string :value
+            t.timestamps
+          end
+        end
+      end
+    end
+  end
+
+  class MigrationContext < ::ActiveRecord::MigrationContext #:nodoc:
+    attr_reader :migrations_paths, :schema_migration
+
+    def initialize(migrations_paths, schema_migration)
+      @migrations_paths = migrations_paths
+      @schema_migration = schema_migration
+    end
+
+    def down(target_version = nil)
+      selected_migrations = if block_given?
+        migrations.select { |m| yield m }
+      else
+        migrations
+      end
+
+      ClickhouseActiverecord::Migrator.new(:down, selected_migrations, schema_migration, target_version).migrate
+    end
+
+    def get_all_versions
+      if schema_migration.table_exists?
+        schema_migration.all_versions.map(&:to_i)
+      else
+        []
+      end
+    end
+
+  end
+
+  class Migrator < ::ActiveRecord::Migrator
+
+    def initialize(direction, migrations, schema_migration, target_version = nil)
+      @direction         = direction
+      @target_version    = target_version
+      @migrated_versions = nil
+      @migrations        = migrations
+      @schema_migration  = schema_migration
+
+      validate(@migrations)
+
+      @schema_migration.create_table
+      ClickhouseActiverecord::InternalMetadata.create_table
+    end
+
+    def record_version_state_after_migrating(version)
+      if down?
+        migrated.delete(version)
+        @schema_migration.create!(version: version.to_s, active: 0)
+      else
+        super
+      end
+    end
+  end
+end

data/lib/clickhouse-activerecord/schema.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module ClickhouseActiverecord
+
+  class Schema < ::ActiveRecord::Schema
+
+    def define(info, &block) # :nodoc:
+      instance_eval(&block)
+
+      if info[:version].present?
+        connection.schema_migration.create_table
+        connection.assume_migrated_upto_version(info[:version], ClickhouseActiverecord::Migrator.migrations_paths)
+      end
+
+      ClickhouseActiverecord::InternalMetadata.create_table
+      ClickhouseActiverecord::InternalMetadata[:environment] = connection.migration_context.current_environment
+    end
+  end
+end

data/lib/clickhouse-activerecord/schema_dumper.rb

@@ -1,10 +1,129 @@
 module ClickhouseActiverecord
   class SchemaDumper < ::ActiveRecord::ConnectionAdapters::SchemaDumper
 
+    attr_accessor :simple
+
+    class << self
+      def dump(connection = ActiveRecord::Base.connection, stream = STDOUT, config = ActiveRecord::Base, default = false)
+        dumper = connection.create_schema_dumper(generate_options(config))
+        dumper.simple = default
+        dumper.dump(stream)
+        stream
+      end
+    end
+
+    private
+
+    def header(stream)
+      stream.puts <<HEADER
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# This file is the source Rails uses to define your schema when running `rails
+# #{simple ? 'db' : 'clickhouse'}:schema:load`. When creating a new database, `rails #{simple ? 'db' : 'clickhouse'}:schema:load` tends to
+# be faster and is potentially less error prone than running all of your
+# migrations from scratch. Old migrations may fail to apply correctly if those
+# migrations use external dependencies or application code.
+#
+# It's strongly recommended that you check this file into your version control system.
+
+#{simple ? 'ActiveRecord' : 'ClickhouseActiverecord'}::Schema.define(#{define_params}) do
+
+HEADER
+    end
+
     def table(table, stream)
-      stream.puts "  # TABLE: #{table}"
-      stream.puts "  # SQL: #{@connection.do_system_execute("SHOW CREATE TABLE #{table.gsub(/^\.inner\./, '')}")['data'].try(:first).try(:first)}"
-      super(table.gsub(/^\.inner\./, ''), stream)
+      if table.match(/^\.inner\./).nil?
+        unless simple
+          stream.puts "  # TABLE: #{table}"
+          sql = @connection.do_system_execute("SHOW CREATE TABLE `#{table.gsub(/^\.inner\./, '')}`")['data'].try(:first).try(:first)
+          stream.puts "  # SQL: #{sql.gsub(/ENGINE = Replicated(.*?)\('[^']+',\s*'[^']+',?\s?([^\)]*)?\)/, "ENGINE = \\1(\\2)")}" if sql
+          # super(table.gsub(/^\.inner\./, ''), stream)
+
+          # detect view table
+          match = sql.match(/^CREATE\s+(MATERIALIZED)\s+VIEW/)
+        end
+
+        # Copy from original dumper
+        columns = @connection.columns(table)
+        begin
+          tbl = StringIO.new
+
+          # first dump primary key column
+          pk = @connection.primary_key(table)
+
+          tbl.print "  create_table #{remove_prefix_and_suffix(table).inspect}"
+
+          unless simple
+            # Add materialize flag
+            tbl.print ', view: true' if match
+            tbl.print ', materialized: true' if match && match[1].presence
+          end
+
+          case pk
+          when String
+            tbl.print ", primary_key: #{pk.inspect}" unless pk == "id"
+            pkcol = columns.detect { |c| c.name == pk }
+            pkcolspec = column_spec_for_primary_key(pkcol)
+            if pkcolspec.present?
+              tbl.print ", #{format_colspec(pkcolspec)}"
+            end
+          when Array
+            tbl.print ", primary_key: #{pk.inspect}"
+          else
+            tbl.print ", id: false"
+          end
+
+          unless simple
+            table_options = @connection.table_options(table)
+            if table_options.present?
+              tbl.print ", #{format_options(table_options)}"
+            end
+          end
+
+          tbl.puts ", force: :cascade do |t|"
+
+          # then dump all non-primary key columns
+          if simple || !match
+            columns.each do |column|
+              raise StandardError, "Unknown type '#{column.sql_type}' for column '#{column.name}'" unless @connection.valid_type?(column.type)
+              next if column.name == pk
+              type, colspec = column_spec(column)
+              tbl.print "    t.#{type} #{column.name.inspect}"
+              tbl.print ", #{format_colspec(colspec)}" if colspec.present?
+              tbl.puts
+            end
+          end
+
+          indexes_in_create(table, tbl)
+
+          tbl.puts "  end"
+          tbl.puts
+
+          tbl.rewind
+          stream.print tbl.read
+        rescue => e
+          stream.puts "# Could not dump table #{table.inspect} because of following #{e.class}"
+          stream.puts "#   #{e.message}"
+          stream.puts
+        end
+      end
+    end
+
+    def format_options(options)
+      if options && options[:options]
+        options[:options] = options[:options].gsub(/^Replicated(.*?)\('[^']+',\s*'[^']+',?\s?([^\)]*)?\)/, "\\1(\\2)")
+      end
+      super
+    end
+
+    def format_colspec(colspec)
+      if simple
+        super.gsub(/CAST\('?([^,']*)'?,\s?'.*?'\)/, "\\1")
+      else
+        super
+      end
     end
   end
 end

data/lib/clickhouse-activerecord/version.rb

@@ -1,3 +1,3 @@
 module ClickhouseActiverecord
-  VERSION = '0.3.12'
+  VERSION = '0.4.5'
 end

data/lib/generators/clickhouse_migration_generator.rb

@@ -12,8 +12,8 @@ class ClickhouseMigrationGenerator < ActiveRecord::Generators::MigrationGenerato
   private
 
   def db_migrate_path
-    if defined?(Rails.application) && Rails.application
-      configured_migrate_path || default_migrate_path
+    if defined?(Rails.application) && Rails.application && respond_to?(:configured_migrate_path, true)
+      configured_migrate_path
     else
       default_migrate_path
     end

data/lib/tasks/clickhouse.rake

@@ -2,6 +2,14 @@
 
 namespace :clickhouse do
 
+  task prepare_schema_migration_table: :environment do
+    ClickhouseActiverecord::SchemaMigration.create_table unless ENV['simple'] || ARGV.map{|a| a.include?('--simple') ? true : nil}.compact.any?
+  end
+
+  task prepare_internal_metadata_table: :environment do
+    ClickhouseActiverecord::InternalMetadata.create_table unless ENV['simple'] || ARGV.map{|a| a.include?('--simple') ? true : nil}.compact.any?
+  end
+
   task load_config: :environment do
     ENV['SCHEMA'] = "db/clickhouse_schema.rb"
     ActiveRecord::Migrator.migrations_paths = ["db/migrate_clickhouse"]
@@ -12,16 +20,18 @@ namespace :clickhouse do
 
     # todo not testing
     desc 'Load database schema'
-    task load: :load_config do
-      load("#{Rails.root}/db/clickhouse_schema.rb")
+    task load: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do |t, args|
+      simple = ENV['simple'] || ARGV.map{|a| a.include?('--simple') ? true : nil}.compact.any? ? '_simple' : nil
+      load("#{Rails.root}/db/clickhouse_schema#{simple}.rb")
     end
 
     desc 'Dump database schema'
-    task dump: :environment do
-      filename = "#{Rails.root}/db/clickhouse_schema.rb"
+    task dump: :environment do |t, args|
+      simple = ENV['simple'] || args[:simple] || ARGV.map{|a| a.include?('--simple') ? true : nil}.compact.any? ? '_simple' : nil
+      filename = "#{Rails.root}/db/clickhouse_schema#{simple}.rb"
       File.open(filename, 'w:utf-8') do |file|
         ActiveRecord::Base.establish_connection(:"#{Rails.env}_clickhouse")
-        ClickhouseActiverecord::SchemaDumper.dump(ActiveRecord::Base.connection, file)
+        ClickhouseActiverecord::SchemaDumper.dump(ActiveRecord::Base.connection, file, ActiveRecord::Base, !!simple)
       end
     end
 
@@ -61,7 +71,15 @@ namespace :clickhouse do
   end
 
   desc 'Migrate the clickhouse database'
-  task migrate: :load_config do
+  task migrate: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do
     Rake::Task['db:migrate'].execute
+    if File.exists? "#{Rails.root}/db/clickhouse_schema_simple.rb"
+      Rake::Task['clickhouse:schema:dump'].execute(simple: true)
+    end
+  end
+
+  desc 'Rollback the clickhouse database'
+  task rollback: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do
+    Rake::Task['db:rollback'].execute
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clickhouse-activerecord
 version: !ruby/object:Gem::Version
-  version: 0.3.12
+  version: 0.4.5
 platform: ruby
 authors:
 - Sergey Odintsov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-14 00:00:00.000000000 Z
+date: 2020-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -122,7 +122,9 @@ files:
 - lib/clickhouse-activerecord.rb
 - lib/clickhouse-activerecord/arel/table.rb
 - lib/clickhouse-activerecord/arel/visitors/to_sql.rb
+- lib/clickhouse-activerecord/migration.rb
 - lib/clickhouse-activerecord/railtie.rb
+- lib/clickhouse-activerecord/schema.rb
 - lib/clickhouse-activerecord/schema_dumper.rb
 - lib/clickhouse-activerecord/tasks.rb
 - lib/clickhouse-activerecord/version.rb