clickhouse-activerecord 0.5.7 → 0.5.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 79b43575e5eef93daa11e1a2aca2984f696cbef59021f55d1470b2dc0a5d1e3d
-  data.tar.gz: b79cccb85d8a07fa4dc533d7b7e9042eeaa2afead26214c075b3131e1843394b
+  metadata.gz: 6534694860bbb41c21789239e0589406bcbef64bc1750645c68485e77acb7310
+  data.tar.gz: c6640ccc74004a93bfaa18d93dd1c135411d66f27273eb65848f9843f64fd6d0
 SHA512:
-  metadata.gz: 840b01cc1d5b88eee5e031fada23cfd35f7a8d948a35a767b2f5baf1596af665f28c56778e699960511004367b16ee3130be1f5bd2ee713a9a0d5053a9b58a37
-  data.tar.gz: 4e613a19c51b362a05634da06d9528809ed09f01ebec4eac9984656156e9b877bdc4ce321e24bea63ae4e49d85609fa9dda781db60159e60b6d6ee06e8a99a3e
+  metadata.gz: f848ad57f80f061aa85a593a90bff65707e8dbe281caf3f4a41914d93a4723351bf7e5ca12bfdc7c8ed2cbed5df3db39472b8d015645875476b68941d7de6111
+  data.tar.gz: 1592560741198e983d43599ff78b145262e0638951a2825dcaac993e0dcc0195bf85980b5eeff80d973e0573aeaa865d6d6c8acdd7ee2b2b1ba6bb6740e734cf

data/CHANGELOG.md

@@ -1,10 +1,16 @@
+### Version 0.5.10 (Jun 22, 2022)
+
+ * Fixes to create_table method (#70)
+ * Added support for rails 7 (#65)
+ * Use ClickHouse default KeepAlive timeout of 10 seconds (#67)
+
 ### Version 0.5.6 (Oct 25, 2021)
- 
+
  * Added auto creating service distributed tables and additional options for creating view [@ygreeek](https://github.com/ygreeek)
  * Added default user agent
 
 ### Version 0.5.3 (Sep 22, 2021)
- 
+
  * Fix replica cluster for a new syntax MergeTree
  * Fix support rails 5.2 on alter table
  * Support array type of column

data/clickhouse-activerecord.gemspec

@@ -26,7 +26,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'bundler', '>= 1.13.4'
   spec.add_runtime_dependency 'activerecord', '>= 5.2'
 
-  spec.add_development_dependency 'bundler', '~> 1.15'
+  spec.add_development_dependency 'bundler', '>= 1.15'
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.4'
   spec.add_development_dependency 'pry', '~> 0.12'

data/core_extensions/active_record/migration/command_recorder.rb

@@ -2,21 +2,12 @@ module CoreExtensions
   module ActiveRecord
     module Migration
       module CommandRecorder
-        def create_table_with_distributed(*args, &block)
-          record(:create_table_with_distributed, args, &block)
-        end
-
         def create_view(*args, &block)
           record(:create_view, args, &block)
         end
 
         private
 
-        def invert_create_table_with_distributed(args)
-          table_name, options = args
-          [:drop_table_with_distributed, table_name, options]
-        end
-
         def invert_create_view(args)
           view_name, options = args
           [:drop_table, view_name, options]

data/lib/active_record/connection_adapters/clickhouse/oid/array.rb

@@ -23,6 +23,42 @@ module ActiveRecord
             @subtype
           end
 
+          def deserialize(value)
+            if value.is_a?(::Array)
+              value.map { |item| deserialize(item) }
+            else
+              return value if value.nil?
+              case @subtype
+                when :integer
+                  value.to_i
+                when :datetime
+                  ::DateTime.parse(value)
+                when :date
+                  ::Date.parse(value)
+              else
+                super
+              end
+            end
+          end
+
+          def serialize(value)
+            if value.is_a?(::Array)
+              value.map { |item| serialize(item) }
+            else
+              return value if value.nil?
+              case @subtype
+                when :integer
+                  value.to_i
+                when :datetime
+                  DateTime.new.serialize(value)
+                when :date
+                  Date.new.serialize(value)
+              else
+                super
+              end
+            end
+          end
+
         end
       end
     end

data/lib/active_record/connection_adapters/clickhouse/oid/date_time.rb

@@ -9,9 +9,8 @@ module ActiveRecord
           def serialize(value)
             value = super
             return unless value
-            return value.strftime('%Y-%m-%d %H:%M:%S') unless value.acts_like?(:time)
 
-            value.to_time.strftime('%Y-%m-%d %H:%M:%S')
+            value.strftime('%Y-%m-%d %H:%M:%S' + (@precision.present? && @precision > 0 ? ".%#{@precision}N" : ''))
           end
 
           def type_cast_from_database(value)

data/lib/active_record/connection_adapters/clickhouse/schema_creation.rb

@@ -19,9 +19,18 @@ module ActiveRecord
         end
 
         def add_column_options!(sql, options)
+          if options[:value]
+            sql.gsub!(/\s+(.*)/, " \\1(#{options[:value]})")
+          end
+          if options[:fixed_string]
+            sql.gsub!(/\s+(.*)/, " FixedString(#{options[:fixed_string]})")
+          end
           if options[:null] || options[:null].nil?
             sql.gsub!(/\s+(.*)/, ' Nullable(\1)')
           end
+          if options[:low_cardinality]
+            sql.gsub!(/\s+(.*)/, ' LowCardinality(\1)')
+          end
           if options[:array]
             sql.gsub!(/\s+(.*)/, ' Array(\1)')
           end
@@ -73,7 +82,7 @@ module ActiveRecord
           return unless match
           return if match[:database]
 
-          create_sql << "TO #{current_database}.#{options.to.sub('.', '')} "
+          create_sql << "TO #{current_database}.#{match[:table_name].sub('.', '')}"
         end
 
         def visit_TableDefinition(o)
@@ -116,7 +125,11 @@ module ActiveRecord
         end
 
         def current_database
-          ActiveRecord::Base.connection_db_config.database
+          if ActiveRecord::version >= Gem::Version.new('6')
+            ActiveRecord::Base.connection_db_config.database
+          else
+            ActiveRecord::Base.connection_config[:database]
+          end
         end
       end
     end

data/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb

@@ -62,6 +62,38 @@ module ActiveRecord
           end
           args.each { |name| column(name, kind, **options.except(:limit, :unsigned)) }
         end
+
+        def datetime(*args, **options)
+          kind = :datetime
+
+          if options[:precision]
+            kind = :datetime64
+            options[:value] = options[:precision]
+          end
+
+          args.each { |name| column(name, kind, **options.except(:precision)) }
+        end
+
+        def uuid(*args, **options)
+          args.each { |name| column(name, :uuid, **options) }
+        end
+
+        def enum(*args, **options)
+          kind = :enum8
+
+          unless options[:value].is_a? Hash
+            raise ArgumentError, "Column #{args.first}: option 'value' must be Hash, got: #{options[:value].class}"
+          end
+
+          options[:value] = options[:value].each_with_object([]) { |(k, v), arr| arr.push("'#{k}' = #{v}") }.join(', ')
+
+          if options[:limit]
+            kind = :enum8  if options[:limit] == 1
+            kind = :enum16 if options[:limit] == 2
+          end
+
+          args.each { |name| column(name, kind, **options.except(:limit)) }
+        end
       end
     end
   end

data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb

@@ -6,8 +6,8 @@ module ActiveRecord
   module ConnectionAdapters
     module Clickhouse
       module SchemaStatements
-        def execute(sql, name = nil)
-          do_execute(sql, name)
+        def execute(sql, name = nil, settings: {})
+          do_execute(sql, name, settings: settings)
         end
 
         def exec_insert(sql, name, _binds, _pk = nil, _sequence_name = nil)
@@ -39,7 +39,7 @@ module ActiveRecord
         end
 
         def tables(name = nil)
-          result = do_system_execute('SHOW TABLES', name)
+          result = do_system_execute("SHOW TABLES WHERE name NOT LIKE '.inner_id.%'", name)
           return [] if result.nil?
           result['data'].flatten
         end
@@ -92,7 +92,7 @@ module ActiveRecord
             if (duplicate = inserting.detect { |v| inserting.count(v) > 1 })
               raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict."
             end
-            execute insert_versions_sql(inserting)
+            do_execute(insert_versions_sql(inserting), nil, settings: {max_partitions_per_insert_block: [100, inserting.size].max})
           end
         end
 

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -11,6 +11,7 @@ require 'active_record/connection_adapters/clickhouse/schema_definitions'
 require 'active_record/connection_adapters/clickhouse/schema_creation'
 require 'active_record/connection_adapters/clickhouse/schema_statements'
 require 'net/http'
+require 'openssl'
 
 module ActiveRecord
   class Base
@@ -32,6 +33,7 @@ module ActiveRecord
             sslca: config[:sslca],
             read_timeout: config[:read_timeout],
             write_timeout: config[:write_timeout],
+            keep_alive_timeout: config[:keep_alive_timeout]
           }
         end
 
@@ -46,21 +48,6 @@ module ActiveRecord
     end
   end
 
-  class Relation
-
-    # Replace for only ClickhouseAdapter
-    def reverse_order!
-      orders = order_values.uniq
-      orders.reject!(&:blank?)
-      if self.connection.is_a?(ConnectionAdapters::ClickhouseAdapter) && orders.empty? && !primary_key
-        self.order_values = %w(date created_at).select {|c| column_names.include?(c) }.map{|c| arel_attribute(c).desc }
-      else
-        self.order_values = reverse_sql_order(orders)
-      end
-      self
-    end
-  end
-
   module TypeCaster
     class Map
       def is_view
@@ -74,7 +61,7 @@ module ActiveRecord
   end
 
   module ModelSchema
-     module ClassMethods
+    module ClassMethods
       def is_view
         @is_view || false
       end
@@ -86,9 +73,8 @@ module ActiveRecord
       def arel_table # :nodoc:
         @arel_table ||= ClickhouseActiverecord::Arel::Table.new(table_name, type_caster: type_caster)
       end
-
     end
-   end
+  end
 
   module ConnectionAdapters
     class ClickhouseColumn < Column
@@ -104,8 +90,13 @@ module ActiveRecord
         float: { name: 'Float32' },
         decimal: { name: 'Decimal' },
         datetime: { name: 'DateTime' },
+        datetime64: { name: 'DateTime64' },
         date: { name: 'Date' },
         boolean: { name: 'UInt8' },
+        uuid: { name: 'UUID' },
+
+        enum8: { name: 'Enum8' },
+        enum16: { name: 'Enum16' },
 
         int8:  { name: 'Int8' },
         int16: { name: 'Int16' },
@@ -182,11 +173,25 @@ module ActiveRecord
         end
       end
 
+      # `extract_scale` and `extract_precision` are the same as in the Rails abstract base class,
+      # except this permits a space after the comma
+
+      def extract_scale(sql_type)
+        case sql_type
+        when /\((\d+)\)/ then 0
+        when /\((\d+)(,\s?(\d+))\)/ then $3.to_i
+        end
+      end
+
+      def extract_precision(sql_type)
+        $1.to_i if sql_type =~ /\((\d+)(,\s?\d+)?\)/
+      end
+
       def initialize_type_map(m) # :nodoc:
         super
         register_class_with_limit m, %r(String), Type::String
         register_class_with_limit m, 'Date',  Clickhouse::OID::Date
-        register_class_with_limit m, 'DateTime',  Clickhouse::OID::DateTime
+        register_class_with_precision m, %r(datetime)i,  Clickhouse::OID::DateTime
 
         register_class_with_limit m, %r(Int8), Type::Integer
         register_class_with_limit m, %r(Int16), Type::Integer
@@ -207,17 +212,34 @@ module ActiveRecord
         end
       end
 
+      def _quote(value)
+        case value
+        when Array
+          '[' + value.map { |v| _quote(v) }.join(', ') + ']'
+        else
+          super
+        end
+      end
+
       # Quoting time without microseconds
       def quoted_date(value)
         if value.acts_like?(:time)
-          zone_conversion_method = ActiveRecord::Base.default_timezone == :utc ? :getutc : :getlocal
+          if ActiveRecord::version >= Gem::Version.new('7')
+            zone_conversion_method = ActiveRecord.default_timezone == :utc ? :getutc : :getlocal
+          else
+            zone_conversion_method = ActiveRecord::Base.default_timezone == :utc ? :getutc : :getlocal
+          end
 
           if value.respond_to?(zone_conversion_method)
             value = value.send(zone_conversion_method)
           end
         end
 
-        value.to_s(:db)
+        if ActiveRecord::version >= Gem::Version.new('7')
+          value.to_fs(:db)
+        else
+          value.to_s(:db)
+        end
       end
 
       def column_name_for_operation(operation, node) # :nodoc:
@@ -269,32 +291,30 @@ module ActiveRecord
           drop_table(table_name, options.merge(if_exists: true))
         end
 
-        execute schema_creation.accept td
+        do_execute(schema_creation.accept(td), format: nil)
       end
 
       def create_table(table_name, **options, &block)
         options = apply_replica(table_name, options)
         td = create_table_definition(apply_cluster(table_name), **options)
         block.call td if block_given?
+        td.column(:id, options[:id], null: false) if options[:id].present? && td[:id].blank?
 
         if options[:force]
           drop_table(table_name, options.merge(if_exists: true))
         end
 
-        execute schema_creation.accept td
-      end
-
-      def create_table_with_distributed(table_name, **options, &block)
-        sharding_key = options.delete(:sharding_key) || 'rand()'
-        create_table("#{table_name}_distributed", **options, &block)
-        raise 'Set a cluster' unless cluster
+        do_execute(schema_creation.accept(td), format: nil)
 
-        distributed_options = "Distributed(#{cluster},#{@config[:database]},#{table_name}_distributed,#{sharding_key})"
-        create_table(table_name, **options.merge(options: distributed_options), &block)
-      end
+        if options[:with_distributed]
+          distributed_table_name = options.delete(:with_distributed)
+          sharding_key = options.delete(:sharding_key) || 'rand()'
+          raise 'Set a cluster' unless cluster
 
-      def drop_table_with_distributed(table_name, **options)
-        ["#{table_name}_distributed", table_name].each { |name| drop_table(name, **options) }
+          distributed_options =
+            "Distributed(#{cluster}, #{@config[:database]}, #{table_name}, #{sharding_key})"
+          create_table(distributed_table_name, **options.merge(options: distributed_options), &block)
+        end
       end
 
       # Drops a ClickHouse database.
@@ -312,10 +332,29 @@ module ActiveRecord
 
       def drop_table(table_name, options = {}) # :nodoc:
         do_execute apply_cluster "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}"
+
+        if options[:with_distributed]
+          distributed_table_name = options.delete(:with_distributed)
+          drop_table(distributed_table_name, **options)
+        end
+      end
+
+      def add_column(table_name, column_name, type, **options)
+        return if options[:if_not_exists] == true && column_exists?(table_name, column_name, type)
+
+        at = create_alter_table table_name
+        at.add_column(column_name, type, **options)
+        execute(schema_creation.accept(at), nil, settings: {wait_end_of_query: 1, send_progress_in_http_headers: 1})
+      end
+
+      def remove_column(table_name, column_name, type = nil, **options)
+        return if options[:if_exists] == true && !column_exists?(table_name, column_name)
+
+        execute("ALTER TABLE #{quote_table_name(table_name)} #{remove_column_for_alter(table_name, column_name, type, **options)}", nil, settings: {wait_end_of_query: 1, send_progress_in_http_headers: 1})
       end
 
       def change_column(table_name, column_name, type, options = {})
-        result = do_execute "ALTER TABLE #{quote_table_name(table_name)} #{change_column_for_alter(table_name, column_name, type, options)}"
+        result = do_execute("ALTER TABLE #{quote_table_name(table_name)} #{change_column_for_alter(table_name, column_name, type, options)}", nil, settings: {wait_end_of_query: 1, send_progress_in_http_headers: 1})
         raise "Error parse json response: #{result}" if result.presence && !result.is_a?(Hash)
       end
 
@@ -351,12 +390,18 @@ module ActiveRecord
 
       def database_engine_atomic?
         current_database_engine = "select engine from system.databases where name = '#{@config[:database]}'"
-        res = ActiveRecord::Base.connection.select_one(current_database_engine)
+        res = select_one(current_database_engine)
         res['engine'] == 'Atomic' if res
       end
 
       def apply_cluster(sql)
-        cluster ? "#{sql} ON CLUSTER #{cluster}" : sql
+        if cluster
+          normalized_cluster_name = cluster.start_with?('{') ? "'#{cluster}'" : cluster
+
+          "#{sql} ON CLUSTER #{normalized_cluster_name}"
+        else
+          sql
+        end
       end
 
       def supports_insert_on_duplicate_skip?
@@ -393,6 +438,9 @@ module ActiveRecord
         @connection.read_timeout = @connection_parameters[:read_timeout] if @connection_parameters[:read_timeout]
         @connection.write_timeout = @connection_parameters[:write_timeout] if @connection_parameters[:write_timeout]
 
+        # Use clickhouse default keep_alive_timeout value of 10, rather than Net::HTTP's default of 2
+        @connection.keep_alive_timeout = @connection_parameters[:keep_alive_timeout] || 10
+
         @connection
       end
 

data/lib/clickhouse-activerecord/migration.rb

@@ -10,16 +10,17 @@ module ClickhouseActiverecord
 
         version_options = connection.internal_string_options_for_primary_key
         table_options = {
-          id: false, options: 'ReplacingMergeTree(ver) PARTITION BY version ORDER BY (version)', if_not_exists: true
+          id: false, options: 'ReplacingMergeTree(ver) ORDER BY (version)', if_not_exists: true
         }
-        if connection.instance_variable_get(:@full_config)[:distributed_service_tables]
-          table_options.merge!(sharding_key: 'cityHash64(version)')
-          table_creation_method = 'create_table_with_distributed'
-        else
-          table_creation_method = 'create_table'
+        full_config = connection.instance_variable_get(:@full_config) || {}
+
+        if full_config[:distributed_service_tables]
+          table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(version)')
+
+          distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}"
         end
 
-        connection.public_send(table_creation_method, table_name, **table_options) do |t|
+        connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t|
           t.string :version, **version_options
           t.column :active, 'Int8', null: false, default: '1'
           t.datetime :ver, null: false, default: -> { 'now()' }
@@ -43,14 +44,15 @@ module ClickhouseActiverecord
           options: connection.adapter_name.downcase == 'clickhouse' ? 'MergeTree() PARTITION BY toDate(created_at) ORDER BY (created_at)' : '',
           if_not_exists: true
         }
-        if connection.instance_variable_get(:@full_config).try(:[], :distributed_service_tables)
-          table_options.merge!(sharding_key: 'cityHash64(created_at)')
-          table_creation_method = 'create_table_with_distributed'
-        else
-          table_creation_method = 'create_table'
+        full_config = connection.instance_variable_get(:@full_config) || {}
+
+        if full_config[:distributed_service_tables]
+          table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(created_at)')
+
+          distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}"
         end
 
-        connection.public_send(table_creation_method, table_name, **table_options) do |t|
+        connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t|
           t.string :key, **key_options
           t.string :value
           t.timestamps

data/lib/clickhouse-activerecord/railtie.rb

@@ -4,6 +4,12 @@ module ClickhouseActiverecord
   require 'rails'
 
   class Railtie < Rails::Railtie
+    initializer "clickhouse.load" do
+      ActiveSupport.on_load :active_record do
+        ClickhouseActiverecord.load
+      end
+    end
+
     rake_tasks { load 'tasks/clickhouse.rake' }
   end
 end

data/lib/clickhouse-activerecord/tasks.rb

@@ -14,7 +14,7 @@ module ClickhouseActiverecord
       connection.create_database @configuration["database"]
     rescue ActiveRecord::StatementInvalid => e
       if e.cause.to_s.include?('already exists')
-        raise ActiveRecord::Tasks::DatabaseAlreadyExists
+        raise ActiveRecord::DatabaseAlreadyExists
       else
         raise
       end

data/lib/clickhouse-activerecord/version.rb

@@ -1,3 +1,3 @@
 module ClickhouseActiverecord
-  VERSION = '0.5.7'
+  VERSION = '0.5.15'
 end

data/lib/clickhouse-activerecord.rb

@@ -2,6 +2,8 @@
 
 require 'active_record/connection_adapters/clickhouse_adapter'
 
+require 'core_extensions/active_record/relation'
+
 require_relative '../core_extensions/active_record/migration/command_recorder'
 ActiveRecord::Migration::CommandRecorder.include CoreExtensions::ActiveRecord::Migration::CommandRecorder
 
@@ -14,5 +16,7 @@ if defined?(Rails::Railtie)
 end
 
 module ClickhouseActiverecord
-
+  def self.load
+    ActiveRecord::Relation.prepend(CoreExtensions::ActiveRecord::Relation)
+  end
 end

data/lib/core_extensions/active_record/relation.rb

@@ -0,0 +1,15 @@
+module CoreExtensions
+  module ActiveRecord
+    module Relation
+      def reverse_order!
+        return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
+
+        orders = order_values.uniq.reject(&:blank?)
+        return super unless orders.empty? && !primary_key
+
+        self.order_values = (column_names & %w[date created_at]).map { |c| arel_table[c].desc }
+        self
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clickhouse-activerecord
 version: !ruby/object:Gem::Version
-  version: 0.5.7
+  version: 0.5.15
 platform: ruby
 authors:
 - Sergey Odintsov
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-25 00:00:00.000000000 Z
+date: 2023-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -42,14 +42,14 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.15'
 - !ruby/object:Gem::Dependency
@@ -130,13 +130,14 @@ files:
 - lib/clickhouse-activerecord/schema_dumper.rb
 - lib/clickhouse-activerecord/tasks.rb
 - lib/clickhouse-activerecord/version.rb
+- lib/core_extensions/active_record/relation.rb
 - lib/generators/clickhouse_migration_generator.rb
 - lib/tasks/clickhouse.rake
 homepage: https://github.com/pnixx/clickhouse-activerecord
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -151,8 +152,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.0.4
+signing_key:
 specification_version: 4
 summary: ClickHouse ActiveRecord
 test_files: []