clickhouse-activerecord 0.4.7 → 0.5.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ed292c2d626374fcb26194a4ca28ca34d60f8c49105b7217bfc7654a5a1b08f0
-  data.tar.gz: 97d4074d47e641e8202fd8796ea049f78bf76d86906ce08877f3eb96b236b116
+  metadata.gz: 79b43575e5eef93daa11e1a2aca2984f696cbef59021f55d1470b2dc0a5d1e3d
+  data.tar.gz: b79cccb85d8a07fa4dc533d7b7e9042eeaa2afead26214c075b3131e1843394b
 SHA512:
-  metadata.gz: 23c4d834c3c3bbaa000b8bd9202064cbabf2c366a8984d1b2fd92dc51e4c6d2811301e39cd8d2b80141581dcedb8cb6647dcbe59d90eb0b0987afdd5f1b22efa
-  data.tar.gz: 0ac017b6614a3072fdad422db66a36201d5e3a62269230f102fbe8b6c7085f48a139cd8c0753adab11ca53a25e91e6eb327298036ebc385eb26f302c4e5178f8
+  metadata.gz: 840b01cc1d5b88eee5e031fada23cfd35f7a8d948a35a767b2f5baf1596af665f28c56778e699960511004367b16ee3130be1f5bd2ee713a9a0d5053a9b58a37
+  data.tar.gz: 4e613a19c51b362a05634da06d9528809ed09f01ebec4eac9984656156e9b877bdc4ce321e24bea63ae4e49d85609fa9dda781db60159e60b6d6ee06e8a99a3e

data/CHANGELOG.md

@@ -1,3 +1,21 @@
+### Version 0.5.6 (Oct 25, 2021)
+ 
+ * Added auto creating service distributed tables and additional options for creating view [@ygreeek](https://github.com/ygreeek)
+ * Added default user agent
+
+### Version 0.5.3 (Sep 22, 2021)
+ 
+ * Fix replica cluster for a new syntax MergeTree
+ * Fix support rails 5.2 on alter table
+ * Support array type of column
+ * Support Rails 6.1.0 [@bdevel](https://github.com/bdevel)
+
+### Version 0.4.10 (Mar 10, 2021)
+
+ * Support ClickHouse 20.9+
+ * Fix schema create / dump
+ * Support all integer types through :limit and :unsigned [@bdevel](https://github.com/bdevel)
+
 ### Version 0.4.4 (Sep 23, 2020)
 
  * Full support migration and rollback database

data/README.md

@@ -1,7 +1,7 @@
 # Clickhouse::Activerecord
 
 A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 5.2.
-Support ClickHouse version from 19.14 LTS.
+Support ClickHouse version from 20.9 LTS.
 
 ## Installation
 
@@ -35,6 +35,21 @@ default: &default
   replica_name: '{replica}' # replica macros name, optional for creating replicated tables
 ```
 
+Alternatively if you wish to pass a custom `Net::HTTP` transport (or any other
+object which supports a `.post()` function with the same parameters as
+`Net::HTTP`'s), you can do this directly instead of specifying
+`host`/`port`/`ssl`:
+
+```ruby
+class ActionView < ActiveRecord::Base
+  establish_connection(
+    adapter: 'clickhouse',
+    database: 'database',
+    connection: Net::HTTP.start('http://example.org', 8123)
+  )
+end
+```
+
 ## Usage in Rails 5
 
 Add your `database.yml` connection information with postfix `_clickhouse` for you environment:
@@ -162,6 +177,45 @@ ActionView.maximum(:date)
 #=> 'Wed, 29 Nov 2017'
 ```
 
+
+### Migration Data Types
+
+Integer types are unsigned by default. Specify signed values with `:unsigned =>
+false`. The default integer is `UInt32`
+
+| Type (bit size)    | Range | :limit (byte size) |
+| :---        |    :----:   |          ---: |
+| Int8 | -128 to 127 | 1 | 
+| Int16 | -32768 to 32767 | 2 |
+| Int32 | -2147483648 to 2,147,483,647 | 3,4 |
+| Int64 | -9223372036854775808 to 9223372036854775807] |  5,6,7,8 |
+| Int128 | ... | 9 - 15 |
+| Int256 | ... | 16+ |
+| UInt8 | 0 to 255 | 1 |
+| UInt16 | 0 to 65,535 | 2 |
+| UInt32 | 0 to 4,294,967,295 | 3,4 |
+| UInt64 | 0 to 18446744073709551615 | 5,6,7,8 |
+| UInt256 | 0 to ... | 8+ |
+| Array | ... | ... |
+
+Example:
+
+``` ruby
+class CreateDataItems < ActiveRecord::Migration
+  def change
+    create_table "data_items", id: false, options: "VersionedCollapsingMergeTree(sign, version) PARTITION BY toYYYYMM(day) ORDER BY category", force: :cascade do |t|
+      t.date "day", null: false
+      t.string "category", null: false
+      t.integer "value_in", null: false
+      t.integer "sign", limit: 1, unsigned: false, default: -> { "CAST(1, 'Int8')" }, null: false
+      t.integer "version", limit: 8, default: -> { "CAST(toUnixTimestamp(now()), 'UInt64')" }, null: false
+    end
+  end
+end
+      
+```
+
+
 ### Using replica and cluster params in connection parameters
 
 ```yml

data/core_extensions/active_record/migration/command_recorder.rb

@@ -0,0 +1,27 @@
+module CoreExtensions
+  module ActiveRecord
+    module Migration
+      module CommandRecorder
+        def create_table_with_distributed(*args, &block)
+          record(:create_table_with_distributed, args, &block)
+        end
+
+        def create_view(*args, &block)
+          record(:create_view, args, &block)
+        end
+
+        private
+
+        def invert_create_table_with_distributed(args)
+          table_name, options = args
+          [:drop_table_with_distributed, table_name, options]
+        end
+
+        def invert_create_view(args)
+          view_name, options = args
+          [:drop_table, view_name, options]
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse/oid/array.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Clickhouse
+      module OID # :nodoc:
+        class Array < Type::Value # :nodoc:
+
+          def initialize(sql_type)
+            @subtype = case sql_type
+                       when /U?Int\d+/
+                         :integer
+                       when /DateTime/
+                         :datetime
+                       when /Date/
+                         :date
+                       else
+                         :string
+            end
+          end
+
+          def type
+            @subtype
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse/schema_creation.rb

@@ -1,9 +1,16 @@
 # frozen_string_literal: true
+begin
+  require "active_record/connection_adapters/deduplicable"
+rescue LoadError => e
+  # Rails < 6.1 does not have this file in this location, ignore
+end
+
+require "active_record/connection_adapters/abstract/schema_creation"
 
 module ActiveRecord
   module ConnectionAdapters
     module Clickhouse
-      class SchemaCreation < AbstractAdapter::SchemaCreation# :nodoc:
+      class SchemaCreation < ConnectionAdapters::SchemaCreation# :nodoc:
 
         def visit_AddColumnDefinition(o)
           sql = +"ADD COLUMN #{accept(o.column)}"
@@ -15,14 +22,23 @@ module ActiveRecord
           if options[:null] || options[:null].nil?
             sql.gsub!(/\s+(.*)/, ' Nullable(\1)')
           end
+          if options[:array]
+            sql.gsub!(/\s+(.*)/, ' Array(\1)')
+          end
           sql.gsub!(/(\sString)\(\d+\)/, '\1')
           sql << " DEFAULT #{quote_default_expression(options[:default], options[:column])}" if options_include_default?(options)
           sql
         end
 
         def add_table_options!(create_sql, options)
-          if options[:options].present?
-            create_sql << " ENGINE = #{options[:options]}"
+          opts = options[:options]
+          if options.respond_to?(:options)
+            # rails 6.1
+            opts ||= options.options
+          end
+
+          if opts.present?
+            create_sql << " ENGINE = #{opts}"
           else
             create_sql << " ENGINE = Log()"
           end
@@ -30,17 +46,48 @@ module ActiveRecord
           create_sql
         end
 
+        def add_as_clause!(create_sql, options)
+          return unless options.as
+
+          assign_database_to_subquery!(options.as) if options.view
+          create_sql << " AS #{to_sql(options.as)}"
+        end
+
+        def assign_database_to_subquery!(subquery)
+          # If you do not specify a database explicitly, ClickHouse will use the "default" database.
+          return unless subquery
+
+          match = subquery.match(/(?<=from)[^.\w]+(?<database>\w+(?=\.))?(?<table_name>[.\w]+)/i)
+          return unless match
+          return if match[:database]
+
+          subquery[match.begin(:table_name)...match.end(:table_name)] =
+            "#{current_database}.#{match[:table_name].sub('.', '')}"
+        end
+
+        def add_to_clause!(create_sql, options)
+          # If you do not specify a database explicitly, ClickHouse will use the "default" database.
+          return unless options.to
+
+          match = options.to.match(/(?<database>.+(?=\.))?(?<table_name>.+)/i)
+          return unless match
+          return if match[:database]
+
+          create_sql << "TO #{current_database}.#{options.to.sub('.', '')} "
+        end
+
         def visit_TableDefinition(o)
           create_sql = +"CREATE#{table_modifier_in_create(o)} #{o.view ? "VIEW" : "TABLE"} "
           create_sql << "IF NOT EXISTS " if o.if_not_exists
           create_sql << "#{quote_table_name(o.name)} "
+          add_to_clause!(create_sql, o) if o.materialized
 
           statements = o.columns.map { |c| accept c }
           statements << accept(o.primary_keys) if o.primary_keys
-
           create_sql << "(#{statements.join(', ')})" if statements.present?
-          add_table_options!(create_sql, table_options(o))
-          create_sql << " AS #{to_sql(o.as)}" if o.as
+          # Attach options for only table or materialized view without TO section
+          add_table_options!(create_sql, o) if !o.view || o.view && o.materialized && !o.to
+          add_as_clause!(create_sql, o)
           create_sql
         end
 
@@ -68,6 +115,9 @@ module ActiveRecord
           change_column_sql
         end
 
+        def current_database
+          ActiveRecord::Base.connection_db_config.database
+        end
       end
     end
   end

data/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb

@@ -5,7 +5,7 @@ module ActiveRecord
     module Clickhouse
       class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition
 
-        attr_reader :view, :materialized, :if_not_exists
+        attr_reader :view, :materialized, :if_not_exists, :to
 
         def initialize(
             conn,
@@ -17,6 +17,7 @@ module ActiveRecord
             comment: nil,
             view: false,
             materialized: false,
+            to: nil,
             **
           )
           @conn = conn
@@ -32,16 +33,35 @@ module ActiveRecord
           @comment = comment
           @view = view || materialized
           @materialized = materialized
+          @to = to
         end
 
         def integer(*args, **options)
-          if options[:limit] == 8
-            args.each { |name| column(name, :big_integer, options.except(:limit)) }
-          else
-            super
+          # default to unsigned
+          unsigned = options[:unsigned]
+          unsigned = true if unsigned.nil?
+
+          kind = :uint32 # default
+
+          if options[:limit]
+            if unsigned
+              kind = :uint8       if options[:limit] == 1
+              kind = :uint16      if options[:limit] == 2
+              kind = :uint32      if [3,4].include?(options[:limit])
+              kind = :uint64      if [5,6,7].include?(options[:limit])
+              kind = :big_integer if options[:limit] == 8
+              kind = :uint256     if options[:limit] > 8
+            else
+              kind = :int8       if options[:limit] == 1
+              kind = :int16      if options[:limit] == 2
+              kind = :int32      if [3,4].include?(options[:limit])
+              kind = :int64     if options[:limit] > 5 && options[:limit] <= 8
+              kind = :int128     if options[:limit] > 8 && options[:limit] <= 16
+              kind = :int256     if options[:limit] > 16
+            end
           end
+          args.each { |name| column(name, kind, **options.except(:limit, :unsigned)) }
         end
-
       end
     end
   end

data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'clickhouse-activerecord/version'
+
 module ActiveRecord
   module ConnectionAdapters
     module Clickhouse
@@ -17,8 +19,15 @@ module ActiveRecord
         def exec_query(sql, name = nil, binds = [], prepare: false)
           result = do_execute(sql, name)
           ActiveRecord::Result.new(result['meta'].map { |m| m['name'] }, result['data'])
-        rescue StandardError => _e
-          raise ActiveRecord::ActiveRecordError, "Response: #{result}"
+        rescue ActiveRecord::ActiveRecordError => e
+          raise e
+        rescue StandardError => e
+          raise ActiveRecord::ActiveRecordError, "Response: #{e.message}"
+        end
+
+        def exec_insert_all(sql, name)
+          do_execute(sql, name, format: nil)
+          true
         end
 
         def exec_update(_sql, _name = nil, _binds = [])
@@ -36,8 +45,8 @@ module ActiveRecord
         end
 
         def table_options(table)
-          sql = do_system_execute("SHOW CREATE TABLE `#{table}`")['data'].try(:first).try(:first)
-          { options: sql.gsub(/^(?:.*?)ENGINE = (.*?)$/, '\\1') }
+          sql = show_create_table(table)
+          { options: sql.gsub(/^(?:.*?)(?:ENGINE = (.*?))?( AS SELECT .*?)?$/, '\\1').presence, as: sql.match(/^CREATE (?:.*?) AS (SELECT .*?)$/).try(:[], 1) }.compact
         end
 
         # Not indexes on clickhouse
@@ -51,7 +60,7 @@ module ActiveRecord
 
         def do_system_execute(sql, name = nil)
           log_with_debug(sql, "#{adapter_name} #{name}") do
-            res = @connection.post("/?#{@config.to_param}", "#{sql} FORMAT JSONCompact")
+            res = @connection.post("/?#{@config.to_param}", "#{sql} FORMAT JSONCompact", 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
 
             process_response(res)
           end
@@ -61,7 +70,7 @@ module ActiveRecord
           log(sql, "#{adapter_name} #{name}") do
             formatted_sql = apply_format(sql, format)
             request_params = @config || {}
-            res = @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql)
+            res = @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
 
             process_response(res)
           end
@@ -114,8 +123,8 @@ module ActiveRecord
           Clickhouse::SchemaCreation.new(self)
         end
 
-        def create_table_definition(*args)
-          Clickhouse::TableDefinition.new(self, *args)
+        def create_table_definition(table_name, **options)
+          Clickhouse::TableDefinition.new(self, table_name, **options)
         end
 
         def new_column_from_field(table_name, field)

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -3,6 +3,7 @@
 require 'clickhouse-activerecord/arel/visitors/to_sql'
 require 'clickhouse-activerecord/arel/table'
 require 'clickhouse-activerecord/migration'
+require 'active_record/connection_adapters/clickhouse/oid/array'
 require 'active_record/connection_adapters/clickhouse/oid/date'
 require 'active_record/connection_adapters/clickhouse/oid/date_time'
 require 'active_record/connection_adapters/clickhouse/oid/big_integer'
@@ -17,9 +18,22 @@ module ActiveRecord
       # Establishes a connection to the database that's used by all Active Record objects
       def clickhouse_connection(config)
         config = config.symbolize_keys
-        host = config[:host] || 'localhost'
-        port = config[:port] || 8123
-        ssl = config[:ssl].present? ? config[:ssl] : port == 443
+
+        if config[:connection]
+          connection = {
+            connection: config[:connection]
+          }
+        else
+          port = config[:port] || 8123
+          connection = {
+            host: config[:host] || 'localhost',
+            port: port,
+            ssl: config[:ssl].present? ? config[:ssl] : port == 443,
+            sslca: config[:sslca],
+            read_timeout: config[:read_timeout],
+            write_timeout: config[:write_timeout],
+          }
+        end
 
         if config.key?(:database)
           database = config[:database]
@@ -27,7 +41,7 @@ module ActiveRecord
           raise ArgumentError, 'No database specified. Missing argument: database.'
         end
 
-        ConnectionAdapters::ClickhouseAdapter.new(logger, [host, port, ssl], { user: config[:username], password: config[:password], database: database }.compact, config)
+        ConnectionAdapters::ClickhouseAdapter.new(logger, connection, { user: config[:username], password: config[:password], database: database }.compact, config)
       end
     end
   end
@@ -50,7 +64,11 @@ module ActiveRecord
   module TypeCaster
     class Map
       def is_view
-        types.is_view
+        if @klass.respond_to?(:is_view)
+          @klass.is_view # rails 6.1
+        else
+          types.is_view # less than 6.1
+        end
       end
     end
   end
@@ -79,7 +97,6 @@ module ActiveRecord
 
     class ClickhouseAdapter < AbstractAdapter
       ADAPTER_NAME = 'Clickhouse'.freeze
-
       NATIVE_DATABASE_TYPES = {
         string: { name: 'String' },
         integer: { name: 'UInt32' },
@@ -88,7 +105,21 @@ module ActiveRecord
         decimal: { name: 'Decimal' },
         datetime: { name: 'DateTime' },
         date: { name: 'Date' },
-        boolean: { name: 'UInt8' }
+        boolean: { name: 'UInt8' },
+
+        int8:  { name: 'Int8' },
+        int16: { name: 'Int16' },
+        int32: { name: 'Int32' },
+        int64:  { name: 'Int64' },
+        int128: { name: 'Int128' },
+        int256: { name: 'Int256' },
+
+        uint8: { name: 'UInt8' },
+        uint16: { name: 'UInt16' },
+        uint32: { name: 'UInt32' },
+        uint64: { name: 'UInt64' },
+        # uint128: { name: 'UInt128' }, not yet implemented in clickhouse
+        uint256: { name: 'UInt256' },
       }.freeze
 
       include Clickhouse::SchemaStatements
@@ -139,10 +170,12 @@ module ActiveRecord
           when /(Nullable)?\(?String\)?/
             super('String')
           when /(Nullable)?\(?U?Int8\)?/
-            super('int2')
-          when /(Nullable)?\(?U?Int(16|32)\)?/
-            super('int4')
-          when /(Nullable)?\(?U?Int(64)\)?/
+            1
+          when /(Nullable)?\(?U?Int16\)?/
+            2
+          when /(Nullable)?\(?U?Int32\)?/
+            nil
+          when /(Nullable)?\(?U?Int64\)?/
             8
           else
             super
@@ -154,14 +187,24 @@ module ActiveRecord
         register_class_with_limit m, %r(String), Type::String
         register_class_with_limit m, 'Date',  Clickhouse::OID::Date
         register_class_with_limit m, 'DateTime',  Clickhouse::OID::DateTime
-        register_class_with_limit m, %r(Uint8), Type::UnsignedInteger
-        m.alias_type 'UInt16', 'UInt8'
-        m.alias_type 'UInt32', 'UInt8'
-        register_class_with_limit m, %r(UInt64), Type::UnsignedInteger
+
         register_class_with_limit m, %r(Int8), Type::Integer
-        m.alias_type 'Int16', 'Int8'
-        m.alias_type 'Int32', 'Int8'
+        register_class_with_limit m, %r(Int16), Type::Integer
+        register_class_with_limit m, %r(Int32), Type::Integer
         register_class_with_limit m, %r(Int64), Type::Integer
+        register_class_with_limit m, %r(Int128), Type::Integer
+        register_class_with_limit m, %r(Int256), Type::Integer
+
+        register_class_with_limit m, %r(UInt8), Type::UnsignedInteger
+        register_class_with_limit m, %r(UInt16), Type::UnsignedInteger
+        register_class_with_limit m, %r(UInt32), Type::UnsignedInteger
+        register_class_with_limit m, %r(UInt64), Type::UnsignedInteger
+        #register_class_with_limit m, %r(UInt128), Type::UnsignedInteger #not implemnted in clickhouse
+        register_class_with_limit m, %r(UInt256), Type::UnsignedInteger
+        # register_class_with_limit m, %r(Array), Clickhouse::OID::Array
+        m.register_type(%r(Array)) do |sql_type|
+          Clickhouse::OID::Array.new(sql_type)
+        end
       end
 
       # Quoting time without microseconds
@@ -201,6 +244,12 @@ module ActiveRecord
         ClickhouseActiverecord::SchemaDumper.create(self, options)
       end
 
+      # @param [String] table
+      # @return [String]
+      def show_create_table(table)
+        do_system_execute("SHOW CREATE TABLE `#{table}`")['data'].try(:first).try(:first).gsub(/[\n\s]+/m, ' ')
+      end
+
       # Create a new ClickHouse database.
       def create_database(name)
         sql = apply_cluster "CREATE DATABASE #{quote_table_name(name)}"
@@ -213,7 +262,7 @@ module ActiveRecord
       def create_view(table_name, **options)
         options.merge!(view: true)
         options = apply_replica(table_name, options)
-        td = create_table_definition(apply_cluster(table_name), options)
+        td = create_table_definition(apply_cluster(table_name), **options)
         yield td if block_given?
 
         if options[:force]
@@ -223,10 +272,10 @@ module ActiveRecord
         execute schema_creation.accept td
       end
 
-      def create_table(table_name, **options)
+      def create_table(table_name, **options, &block)
         options = apply_replica(table_name, options)
-        td = create_table_definition(apply_cluster(table_name), options)
-        yield td if block_given?
+        td = create_table_definition(apply_cluster(table_name), **options)
+        block.call td if block_given?
 
         if options[:force]
           drop_table(table_name, options.merge(if_exists: true))
@@ -235,6 +284,19 @@ module ActiveRecord
         execute schema_creation.accept td
       end
 
+      def create_table_with_distributed(table_name, **options, &block)
+        sharding_key = options.delete(:sharding_key) || 'rand()'
+        create_table("#{table_name}_distributed", **options, &block)
+        raise 'Set a cluster' unless cluster
+
+        distributed_options = "Distributed(#{cluster},#{@config[:database]},#{table_name}_distributed,#{sharding_key})"
+        create_table(table_name, **options.merge(options: distributed_options), &block)
+      end
+
+      def drop_table_with_distributed(table_name, **options)
+        ["#{table_name}_distributed", table_name].each { |name| drop_table(name, **options) }
+      end
+
       # Drops a ClickHouse database.
       def drop_database(name) #:nodoc:
         sql = apply_cluster "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
@@ -275,14 +337,41 @@ module ActiveRecord
         @full_config[:replica_name]
       end
 
+      def use_default_replicated_merge_tree_params?
+        database_engine_atomic? && @full_config[:use_default_replicated_merge_tree_params]
+      end
+
+      def use_replica?
+        (replica || use_default_replicated_merge_tree_params?) && cluster
+      end
+
       def replica_path(table)
         "/clickhouse/tables/#{cluster}/#{@config[:database]}.#{table}"
       end
 
+      def database_engine_atomic?
+        current_database_engine = "select engine from system.databases where name = '#{@config[:database]}'"
+        res = ActiveRecord::Base.connection.select_one(current_database_engine)
+        res['engine'] == 'Atomic' if res
+      end
+
       def apply_cluster(sql)
         cluster ? "#{sql} ON CLUSTER #{cluster}" : sql
       end
 
+      def supports_insert_on_duplicate_skip?
+        true
+      end
+
+      def supports_insert_on_duplicate_update?
+        true
+      end
+
+      def build_insert_sql(insert) # :nodoc:
+        sql = +"INSERT #{insert.into} #{insert.values_list}"
+        sql
+      end
+
       protected
 
       def last_inserted_id(result)
@@ -298,18 +387,36 @@ module ActiveRecord
       private
 
       def connect
-        @connection = Net::HTTP.start(@connection_parameters[0], @connection_parameters[1], use_ssl: @connection_parameters[2], verify_mode: OpenSSL::SSL::VERIFY_NONE)
+        @connection = @connection_parameters[:connection] || Net::HTTP.start(@connection_parameters[:host], @connection_parameters[:port], use_ssl: @connection_parameters[:ssl], verify_mode: OpenSSL::SSL::VERIFY_NONE)
+
+        @connection.ca_file = @connection_parameters[:ca_file] if @connection_parameters[:ca_file]
+        @connection.read_timeout = @connection_parameters[:read_timeout] if @connection_parameters[:read_timeout]
+        @connection.write_timeout = @connection_parameters[:write_timeout] if @connection_parameters[:write_timeout]
+
+        @connection
       end
 
       def apply_replica(table, options)
-        if replica && cluster && options[:options]
-          match = options[:options].match(/^(.*?MergeTree)\(([^\)]*)\)(.*?)$/)
-          if match
-            options[:options] = "Replicated#{match[1]}(#{([replica_path(table), replica].map{|v| "'#{v}'"} + [match[2].presence]).compact.join(', ')})#{match[3]}"
+        if use_replica? && options[:options]
+          if options[:options].match(/^Replicated/)
+            raise 'Do not try create Replicated table. It will be configured based on the *MergeTree engine.'
           end
+
+          options[:options] = configure_replica(table, options[:options])
         end
         options
       end
+
+      def configure_replica(table, options)
+        match = options.match(/^(.*?MergeTree)(?:\(([^\)]*)\))?((?:.|\n)*)/)
+        return options unless match
+
+        if replica
+          engine_params = ([replica_path(table), replica].map { |v| "'#{v}'" } + [match[2].presence]).compact.join(', ')
+        end
+
+        "Replicated#{match[1]}(#{engine_params})#{match[3]}"
+      end
     end
   end
 end

data/lib/clickhouse-activerecord/migration.rb

@@ -1,17 +1,28 @@
+require 'active_record/migration'
+
 module ClickhouseActiverecord
 
   class SchemaMigration < ::ActiveRecord::SchemaMigration
     class << self
 
       def create_table
-        unless table_exists?
-          version_options = connection.internal_string_options_for_primary_key
-
-          connection.create_table(table_name, id: false, options: 'ReplacingMergeTree(ver) PARTITION BY version ORDER BY (version)', if_not_exists: true) do |t|
-            t.string :version, version_options
-            t.column :active, 'Int8', null: false, default: '1'
-            t.datetime :ver, null: false, default: -> { 'now()' }
-          end
+        return if table_exists?
+
+        version_options = connection.internal_string_options_for_primary_key
+        table_options = {
+          id: false, options: 'ReplacingMergeTree(ver) PARTITION BY version ORDER BY (version)', if_not_exists: true
+        }
+        if connection.instance_variable_get(:@full_config)[:distributed_service_tables]
+          table_options.merge!(sharding_key: 'cityHash64(version)')
+          table_creation_method = 'create_table_with_distributed'
+        else
+          table_creation_method = 'create_table'
+        end
+
+        connection.public_send(table_creation_method, table_name, **table_options) do |t|
+          t.string :version, **version_options
+          t.column :active, 'Int8', null: false, default: '1'
+          t.datetime :ver, null: false, default: -> { 'now()' }
         end
       end
 
@@ -24,14 +35,25 @@ module ClickhouseActiverecord
   class InternalMetadata < ::ActiveRecord::InternalMetadata
     class << self
       def create_table
-        unless table_exists?
-          key_options = connection.internal_string_options_for_primary_key
-
-          connection.create_table(table_name, id: false, options: 'MergeTree() PARTITION BY toDate(created_at) ORDER BY (created_at)', if_not_exists: true) do |t|
-            t.string :key, key_options
-            t.string :value
-            t.timestamps
-          end
+        return if table_exists?
+
+        key_options = connection.internal_string_options_for_primary_key
+        table_options = {
+          id: false,
+          options: connection.adapter_name.downcase == 'clickhouse' ? 'MergeTree() PARTITION BY toDate(created_at) ORDER BY (created_at)' : '',
+          if_not_exists: true
+        }
+        if connection.instance_variable_get(:@full_config).try(:[], :distributed_service_tables)
+          table_options.merge!(sharding_key: 'cityHash64(created_at)')
+          table_creation_method = 'create_table_with_distributed'
+        else
+          table_creation_method = 'create_table'
+        end
+
+        connection.public_send(table_creation_method, table_name, **table_options) do |t|
+          t.string :key, **key_options
+          t.string :value
+          t.timestamps
         end
       end
     end
@@ -45,6 +67,16 @@ module ClickhouseActiverecord
       @schema_migration = schema_migration
     end
 
+    def up(target_version = nil)
+      selected_migrations = if block_given?
+        migrations.select { |m| yield m }
+      else
+        migrations
+      end
+
+      ClickhouseActiverecord::Migrator.new(:up, selected_migrations, schema_migration, target_version).migrate
+    end
+
     def down(target_version = nil)
       selected_migrations = if block_given?
         migrations.select { |m| yield m }

data/lib/clickhouse-activerecord/schema_dumper.rb

@@ -33,16 +33,24 @@ module ClickhouseActiverecord
 HEADER
     end
 
+    def tables(stream)
+      sorted_tables = @connection.tables.sort {|a,b| @connection.show_create_table(a).match(/^CREATE\s+(MATERIALIZED\s+)?VIEW/) ? 1 : a <=> b }
+
+      sorted_tables.each do |table_name|
+        table(table_name, stream) unless ignored?(table_name)
+      end
+    end
+
     def table(table, stream)
-      if table.match(/^\.inner\./).nil?
+      if table.match(/^\.inner/).nil?
         unless simple
           stream.puts "  # TABLE: #{table}"
-          sql = @connection.do_system_execute("SHOW CREATE TABLE `#{table.gsub(/^\.inner\./, '')}`")['data'].try(:first).try(:first)
+          sql = @connection.show_create_table(table)
           stream.puts "  # SQL: #{sql.gsub(/ENGINE = Replicated(.*?)\('[^']+',\s*'[^']+',?\s?([^\)]*)?\)/, "ENGINE = \\1(\\2)")}" if sql
           # super(table.gsub(/^\.inner\./, ''), stream)
 
           # detect view table
-          match = sql.match(/^CREATE\s+(MATERIALIZED)\s+VIEW/)
+          match = sql.match(/^CREATE\s+(MATERIALIZED\s+)?VIEW/)
         end
 
         # Copy from original dumper
@@ -125,5 +133,26 @@ HEADER
         super
       end
     end
+
+    def schema_limit(column)
+      return nil if column.type == :float
+      super
+    end
+
+    def schema_unsigned(column)
+      return nil unless column.type == :integer && !simple
+      (column.sql_type =~ /(Nullable)?\(?UInt\d+\)?/).nil? ? false : nil
+    end
+
+    def schema_array(column)
+      (column.sql_type =~ /Array?\(/).nil? ? nil : true
+    end
+
+    def prepare_column_options(column)
+      spec = {}
+      spec[:unsigned] = schema_unsigned(column)
+      spec[:array] = schema_array(column)
+      spec.merge(super).compact
+    end
   end
 end

data/lib/clickhouse-activerecord/tasks.rb

@@ -6,7 +6,7 @@ module ClickhouseActiverecord
     delegate :connection, :establish_connection, :clear_active_connections!, to: ActiveRecord::Base
 
     def initialize(configuration)
-      @configuration = configuration
+      @configuration = configuration.with_indifferent_access
     end
 
     def create
@@ -43,7 +43,15 @@ module ClickhouseActiverecord
     end
 
     def structure_load(*args)
-      File.read(args.first).split(";\n\n").each { |sql| connection.execute(sql) }
+      File.read(args.first).split(";\n\n").each do |sql|
+        if sql.gsub(/[a-z]/i, '').blank?
+          next
+        elsif sql =~ /^INSERT INTO/
+          connection.do_execute(sql, nil, format: nil)
+        else
+          connection.execute(sql)
+        end
+      end
     end
 
     def migrate

data/lib/clickhouse-activerecord/version.rb

@@ -1,3 +1,3 @@
 module ClickhouseActiverecord
-  VERSION = '0.4.7'
+  VERSION = '0.5.7'
 end

data/lib/clickhouse-activerecord.rb

@@ -2,6 +2,9 @@
 
 require 'active_record/connection_adapters/clickhouse_adapter'
 
+require_relative '../core_extensions/active_record/migration/command_recorder'
+ActiveRecord::Migration::CommandRecorder.include CoreExtensions::ActiveRecord::Migration::CommandRecorder
+
 if defined?(Rails::Railtie)
   require 'clickhouse-activerecord/railtie'
   require 'clickhouse-activerecord/schema'

data/lib/generators/clickhouse_migration_generator.rb

@@ -13,7 +13,7 @@ class ClickhouseMigrationGenerator < ActiveRecord::Generators::MigrationGenerato
 
   def db_migrate_path
     if defined?(Rails.application) && Rails.application && respond_to?(:configured_migrate_path, true)
-      configured_migrate_path
+      configured_migrate_path || default_migrate_path
     else
       default_migrate_path
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clickhouse-activerecord
 version: !ruby/object:Gem::Version
-  version: 0.4.7
+  version: 0.5.7
 platform: ruby
 authors:
 - Sergey Odintsov
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-25 00:00:00.000000000 Z
+date: 2021-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -112,6 +112,8 @@ files:
 - bin/console
 - bin/setup
 - clickhouse-activerecord.gemspec
+- core_extensions/active_record/migration/command_recorder.rb
+- lib/active_record/connection_adapters/clickhouse/oid/array.rb
 - lib/active_record/connection_adapters/clickhouse/oid/big_integer.rb
 - lib/active_record/connection_adapters/clickhouse/oid/date.rb
 - lib/active_record/connection_adapters/clickhouse/oid/date_time.rb
@@ -134,7 +136,7 @@ homepage: https://github.com/pnixx/clickhouse-activerecord
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -149,8 +151,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
-signing_key:
+rubygems_version: 3.0.3
+signing_key: 
 specification_version: 4
 summary: ClickHouse ActiveRecord
 test_files: []