clickhouse-activerecord 0.5.2 → 0.5.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c75ca45e5f91a7d83d79b8583539e66021589fc76c7ce48a68f07f49176a6f5a
-  data.tar.gz: 282fba24c4ecacdb11e4d358db15ba88beb0457099f3b9efe22d6c873676ead8
+  metadata.gz: c8f8d01291f12197d0b5d0b8f791631fe144cdc5d855de9dd9c8498e170fafdf
+  data.tar.gz: 2ebd5b6068a179a7c87679db0fe9c34bbe2f9197c6230da9aad1a6935fc9c7d9
 SHA512:
-  metadata.gz: 69b20ad39aced0dc3ba4f6883664dabdba45c8c677848010b1dd926e26de36a707839cbc5d806e59f0bc73a786daeee6c3448681313de0f973d93faa16f63a5d
-  data.tar.gz: cf195bf822304c31f3cd5121f3ca5ea0f5d5fc1b0211862e1cde9ffae32042960ad512a9c10a3a17d146205308aefdfcc09f5de95b07bab952baa9cffab4b60c
+  metadata.gz: 94abbef81ff0000ecbc05092dd3558dc90e53223fd0025c5a57549f4b5f98431a01ee9fceee5587887c9fa6fcc3411420e936ad942150caa8edc5146c7db54df
+  data.tar.gz: '03871bfe2bec394c9612928229cc3380615acd795b25c058072a3aec62a7852e06711f2811286438ab278c838d4ed65736bd47ae449bc0ca5bbad6763692a927'

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+### Version 0.5.6 (Oct 25, 2021)
+ 
+ * Added auto creating service distributed tables and additional options for creating view [@ygreeek](https://github.com/ygreeek)
+ * Added default user agent
+
+### Version 0.5.3 (Sep 22, 2021)
+ 
+ * Fix replica cluster for a new syntax MergeTree
+ * Fix support rails 5.2 on alter table
+ * Support array type of column
+ * Support Rails 6.1.0 [@bdevel](https://github.com/bdevel)
+
 ### Version 0.4.10 (Mar 10, 2021)
 
  * Support ClickHouse 20.9+

data/README.md

@@ -196,6 +196,7 @@ false`. The default integer is `UInt32`
 | UInt32 | 0 to 4,294,967,295 | 3,4 |
 | UInt64 | 0 to 18446744073709551615 | 5,6,7,8 |
 | UInt256 | 0 to ... | 8+ |
+| Array | ... | ... |
 
 Example:
 

data/clickhouse-activerecord.gemspec

@@ -26,7 +26,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'bundler', '>= 1.13.4'
   spec.add_runtime_dependency 'activerecord', '>= 5.2'
 
-  spec.add_development_dependency 'bundler', '~> 1.15'
+  spec.add_development_dependency 'bundler', '>= 1.15'
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.4'
   spec.add_development_dependency 'pry', '~> 0.12'

data/core_extensions/active_record/migration/command_recorder.rb

@@ -0,0 +1,18 @@
+module CoreExtensions
+  module ActiveRecord
+    module Migration
+      module CommandRecorder
+        def create_view(*args, &block)
+          record(:create_view, args, &block)
+        end
+
+        private
+
+        def invert_create_view(args)
+          view_name, options = args
+          [:drop_table, view_name, options]
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse/schema_creation.rb

@@ -19,9 +19,18 @@ module ActiveRecord
         end
 
         def add_column_options!(sql, options)
+          if options[:value]
+            sql.gsub!(/\s+(.*)/, " \\1(#{options[:value]})")
+          end
+          if options[:fixed_string]
+            sql.gsub!(/\s+(.*)/, " FixedString(#{options[:fixed_string]})")
+          end
           if options[:null] || options[:null].nil?
             sql.gsub!(/\s+(.*)/, ' Nullable(\1)')
           end
+          if options[:low_cardinality]
+            sql.gsub!(/\s+(.*)/, ' LowCardinality(\1)')
+          end
           if options[:array]
             sql.gsub!(/\s+(.*)/, ' Array(\1)')
           end
@@ -46,17 +55,48 @@ module ActiveRecord
           create_sql
         end
 
+        def add_as_clause!(create_sql, options)
+          return unless options.as
+
+          assign_database_to_subquery!(options.as) if options.view
+          create_sql << " AS #{to_sql(options.as)}"
+        end
+
+        def assign_database_to_subquery!(subquery)
+          # If you do not specify a database explicitly, ClickHouse will use the "default" database.
+          return unless subquery
+
+          match = subquery.match(/(?<=from)[^.\w]+(?<database>\w+(?=\.))?(?<table_name>[.\w]+)/i)
+          return unless match
+          return if match[:database]
+
+          subquery[match.begin(:table_name)...match.end(:table_name)] =
+            "#{current_database}.#{match[:table_name].sub('.', '')}"
+        end
+
+        def add_to_clause!(create_sql, options)
+          # If you do not specify a database explicitly, ClickHouse will use the "default" database.
+          return unless options.to
+
+          match = options.to.match(/(?<database>.+(?=\.))?(?<table_name>.+)/i)
+          return unless match
+          return if match[:database]
+
+          create_sql << "TO #{current_database}.#{match[:table_name].sub('.', '')}"
+        end
+
         def visit_TableDefinition(o)
           create_sql = +"CREATE#{table_modifier_in_create(o)} #{o.view ? "VIEW" : "TABLE"} "
           create_sql << "IF NOT EXISTS " if o.if_not_exists
           create_sql << "#{quote_table_name(o.name)} "
+          add_to_clause!(create_sql, o) if o.materialized
 
           statements = o.columns.map { |c| accept c }
           statements << accept(o.primary_keys) if o.primary_keys
           create_sql << "(#{statements.join(', ')})" if statements.present?
-          # Attach options for only table or materialized view
-          add_table_options!(create_sql, o)  if !o.view || o.view && o.materialized
-          create_sql << " AS #{to_sql(o.as)}" if o.as
+          # Attach options for only table or materialized view without TO section
+          add_table_options!(create_sql, o) if !o.view || o.view && o.materialized && !o.to
+          add_as_clause!(create_sql, o)
           create_sql
         end
 
@@ -84,6 +124,9 @@ module ActiveRecord
           change_column_sql
         end
 
+        def current_database
+          ActiveRecord::Base.connection_db_config.database
+        end
       end
     end
   end

data/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb

@@ -5,7 +5,7 @@ module ActiveRecord
     module Clickhouse
       class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition
 
-        attr_reader :view, :materialized, :if_not_exists
+        attr_reader :view, :materialized, :if_not_exists, :to
 
         def initialize(
             conn,
@@ -17,6 +17,7 @@ module ActiveRecord
             comment: nil,
             view: false,
             materialized: false,
+            to: nil,
             **
           )
           @conn = conn
@@ -32,6 +33,7 @@ module ActiveRecord
           @comment = comment
           @view = view || materialized
           @materialized = materialized
+          @to = to
         end
 
         def integer(*args, **options)
@@ -58,7 +60,39 @@ module ActiveRecord
               kind = :int256     if options[:limit] > 16
             end
           end
-          args.each { |name| column(name, kind, options.except(:limit, :unsigned)) }
+          args.each { |name| column(name, kind, **options.except(:limit, :unsigned)) }
+        end
+
+        def datetime(*args, **options)
+          kind = :datetime
+
+          if options[:precision]
+            kind = :datetime64
+            options[:value] = options[:precision]
+          end
+
+          args.each { |name| column(name, kind, **options.except(:precision)) }
+        end
+
+        def uuid(*args, **options)
+          args.each { |name| column(name, :uuid, **options) }
+        end
+
+        def enum(*args, **options)
+          kind = :enum8
+
+          unless options[:value].is_a? Hash
+            raise ArgumentError, "Column #{args.first}: option 'value' must be Hash, got: #{options[:value].class}"
+          end
+
+          options[:value] = options[:value].each_with_object([]) { |(k, v), arr| arr.push("'#{k}' = #{v}") }.join(', ')
+
+          if options[:limit]
+            kind = :enum8  if options[:limit] == 1
+            kind = :enum16 if options[:limit] == 2
+          end
+
+          args.each { |name| column(name, kind, **options.except(:limit)) }
         end
       end
     end

data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'clickhouse-activerecord/version'
+
 module ActiveRecord
   module ConnectionAdapters
     module Clickhouse
@@ -37,7 +39,7 @@ module ActiveRecord
         end
 
         def tables(name = nil)
-          result = do_system_execute('SHOW TABLES', name)
+          result = do_system_execute("SHOW TABLES WHERE name NOT LIKE '.inner_id.%'", name)
           return [] if result.nil?
           result['data'].flatten
         end
@@ -58,7 +60,7 @@ module ActiveRecord
 
         def do_system_execute(sql, name = nil)
           log_with_debug(sql, "#{adapter_name} #{name}") do
-            res = @connection.post("/?#{@config.to_param}", "#{sql} FORMAT JSONCompact")
+            res = @connection.post("/?#{@config.to_param}", "#{sql} FORMAT JSONCompact", 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
 
             process_response(res)
           end
@@ -68,7 +70,7 @@ module ActiveRecord
           log(sql, "#{adapter_name} #{name}") do
             formatted_sql = apply_format(sql, format)
             request_params = @config || {}
-            res = @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql)
+            res = @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
 
             process_response(res)
           end

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -46,20 +46,18 @@ module ActiveRecord
     end
   end
 
-  class Relation
-
-    # Replace for only ClickhouseAdapter
+  module ClickhouseRelationReverseOrder
     def reverse_order!
-      orders = order_values.uniq
-      orders.reject!(&:blank?)
-      if self.connection.is_a?(ConnectionAdapters::ClickhouseAdapter) && orders.empty? && !primary_key
-        self.order_values = %w(date created_at).select {|c| column_names.include?(c) }.map{|c| arel_attribute(c).desc }
-      else
-        self.order_values = reverse_sql_order(orders)
-      end
+      return super unless connection.is_a?(ConnectionAdapters::ClickhouseAdapter)
+
+      orders = order_values.uniq.compact_blank
+      return super unless orders.empty? && !primary_key
+
+      self.order_values = %w(date created_at).select {|c| column_names.include?(c) }.map{|c| arel_attribute(c).desc }
       self
     end
   end
+  Relation.prepend(ClickhouseRelationReverseOrder)
 
   module TypeCaster
     class Map
@@ -82,13 +80,19 @@ module ActiveRecord
       def is_view=(value)
         @is_view = value
       end
+    end
+  end
 
-      def arel_table # :nodoc:
-        @arel_table ||= ClickhouseActiverecord::Arel::Table.new(table_name, type_caster: type_caster)
-      end
-
+  ActiveRecord::Core::ClassMethods.module_eval do
+    def arel_table
+      @arel_table ||=
+        if self.connection.is_a?(ConnectionAdapters::ClickhouseAdapter)
+          ClickhouseActiverecord::Arel::Table.new(table_name, type_caster: type_caster)
+        else
+          Arel::Table.new(table_name, klass: self)
+        end
     end
-   end
+  end
 
   module ConnectionAdapters
     class ClickhouseColumn < Column
@@ -104,8 +108,13 @@ module ActiveRecord
         float: { name: 'Float32' },
         decimal: { name: 'Decimal' },
         datetime: { name: 'DateTime' },
+        datetime64: { name: 'DateTime64' },
         date: { name: 'Date' },
         boolean: { name: 'UInt8' },
+        uuid: { name: 'UUID' },
+
+        enum8: { name: 'Enum8' },
+        enum16: { name: 'Enum16' },
 
         int8:  { name: 'Int8' },
         int16: { name: 'Int16' },
@@ -182,6 +191,20 @@ module ActiveRecord
         end
       end
 
+      # `extract_scale` and `extract_precision` are the same as in the Rails abstract base class,
+      # except this permits a space after the comma
+
+      def extract_scale(sql_type)
+        case sql_type
+        when /\((\d+)\)/ then 0
+        when /\((\d+)(,\s?(\d+))\)/ then $3.to_i
+        end
+      end
+
+      def extract_precision(sql_type)
+        $1.to_i if sql_type =~ /\((\d+)(,\s?\d+)?\)/
+      end
+
       def initialize_type_map(m) # :nodoc:
         super
         register_class_with_limit m, %r(String), Type::String
@@ -262,7 +285,7 @@ module ActiveRecord
       def create_view(table_name, **options)
         options.merge!(view: true)
         options = apply_replica(table_name, options)
-        td = create_table_definition(apply_cluster(table_name), options)
+        td = create_table_definition(apply_cluster(table_name), **options)
         yield td if block_given?
 
         if options[:force]
@@ -272,16 +295,26 @@ module ActiveRecord
         execute schema_creation.accept td
       end
 
-      def create_table(table_name, **options)
+      def create_table(table_name, **options, &block)
         options = apply_replica(table_name, options)
-        td = create_table_definition(apply_cluster(table_name), options)
-        yield td if block_given?
+        td = create_table_definition(apply_cluster(table_name), **options)
+        block.call td if block_given?
 
         if options[:force]
           drop_table(table_name, options.merge(if_exists: true))
         end
 
         execute schema_creation.accept td
+
+        if options[:with_distributed]
+          distributed_table_name = options.delete(:with_distributed)
+          sharding_key = options.delete(:sharding_key) || 'rand()'
+          raise 'Set a cluster' unless cluster
+
+          distributed_options =
+            "Distributed(#{cluster}, #{@config[:database]}, #{table_name}, #{sharding_key})"
+          create_table(distributed_table_name, **options.merge(options: distributed_options), &block)
+        end
       end
 
       # Drops a ClickHouse database.
@@ -299,6 +332,11 @@ module ActiveRecord
 
       def drop_table(table_name, options = {}) # :nodoc:
         do_execute apply_cluster "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}"
+
+        if options[:with_distributed]
+          distributed_table_name = options.delete(:with_distributed)
+          drop_table(distributed_table_name, **options)
+        end
       end
 
       def change_column(table_name, column_name, type, options = {})
@@ -324,10 +362,24 @@ module ActiveRecord
         @full_config[:replica_name]
       end
 
+      def use_default_replicated_merge_tree_params?
+        database_engine_atomic? && @full_config[:use_default_replicated_merge_tree_params]
+      end
+
+      def use_replica?
+        (replica || use_default_replicated_merge_tree_params?) && cluster
+      end
+
       def replica_path(table)
         "/clickhouse/tables/#{cluster}/#{@config[:database]}.#{table}"
       end
 
+      def database_engine_atomic?
+        current_database_engine = "select engine from system.databases where name = '#{@config[:database]}'"
+        res = ActiveRecord::Base.connection.select_one(current_database_engine)
+        res['engine'] == 'Atomic' if res
+      end
+
       def apply_cluster(sql)
         cluster ? "#{sql} ON CLUSTER #{cluster}" : sql
       end
@@ -370,14 +422,26 @@ module ActiveRecord
       end
 
       def apply_replica(table, options)
-        if replica && cluster && options[:options]
-          match = options[:options].match(/^(.*?MergeTree)\(([^\)]*)\)(.*?)$/)
-          if match
-            options[:options] = "Replicated#{match[1]}(#{([replica_path(table), replica].map{|v| "'#{v}'"} + [match[2].presence]).compact.join(', ')})#{match[3]}"
+        if use_replica? && options[:options]
+          if options[:options].match(/^Replicated/)
+            raise 'Do not try create Replicated table. It will be configured based on the *MergeTree engine.'
           end
+
+          options[:options] = configure_replica(table, options[:options])
         end
         options
       end
+
+      def configure_replica(table, options)
+        match = options.match(/^(.*?MergeTree)(?:\(([^\)]*)\))?((?:.|\n)*)/)
+        return options unless match
+
+        if replica
+          engine_params = ([replica_path(table), replica].map { |v| "'#{v}'" } + [match[2].presence]).compact.join(', ')
+        end
+
+        "Replicated#{match[1]}(#{engine_params})#{match[3]}"
+      end
     end
   end
 end

data/lib/clickhouse-activerecord/migration.rb

@@ -6,14 +6,24 @@ module ClickhouseActiverecord
     class << self
 
       def create_table
-        unless table_exists?
-          version_options = connection.internal_string_options_for_primary_key
-
-          connection.create_table(table_name, id: false, options: 'ReplacingMergeTree(ver) PARTITION BY version ORDER BY (version)', if_not_exists: true) do |t|
-            t.string :version, **version_options
-            t.column :active, 'Int8', null: false, default: '1'
-            t.datetime :ver, null: false, default: -> { 'now()' }
-          end
+        return if table_exists?
+
+        version_options = connection.internal_string_options_for_primary_key
+        table_options = {
+          id: false, options: 'ReplacingMergeTree(ver) PARTITION BY version ORDER BY (version)', if_not_exists: true
+        }
+        full_config = connection.instance_variable_get(:@full_config) || {}
+
+        if full_config[:distributed_service_tables]
+          table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(version)')
+
+          distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}"
+        end
+
+        connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t|
+          t.string :version, **version_options
+          t.column :active, 'Int8', null: false, default: '1'
+          t.datetime :ver, null: false, default: -> { 'now()' }
         end
       end
 
@@ -26,14 +36,26 @@ module ClickhouseActiverecord
   class InternalMetadata < ::ActiveRecord::InternalMetadata
     class << self
       def create_table
-        unless table_exists?
-          key_options = connection.internal_string_options_for_primary_key
-
-          connection.create_table(table_name, id: false, options: connection.adapter_name.downcase == 'clickhouse' ? 'MergeTree() PARTITION BY toDate(created_at) ORDER BY (created_at)' : '', if_not_exists: true) do |t|
-            t.string :key, key_options
-            t.string :value
-            t.timestamps
-          end
+        return if table_exists?
+
+        key_options = connection.internal_string_options_for_primary_key
+        table_options = {
+          id: false,
+          options: connection.adapter_name.downcase == 'clickhouse' ? 'MergeTree() PARTITION BY toDate(created_at) ORDER BY (created_at)' : '',
+          if_not_exists: true
+        }
+        full_config = connection.instance_variable_get(:@full_config) || {}
+
+        if full_config[:distributed_service_tables]
+          table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(created_at)')
+
+          distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}"
+        end
+
+        connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t|
+          t.string :key, **key_options
+          t.string :value
+          t.timestamps
         end
       end
     end
@@ -47,6 +69,16 @@ module ClickhouseActiverecord
       @schema_migration = schema_migration
     end
 
+    def up(target_version = nil)
+      selected_migrations = if block_given?
+        migrations.select { |m| yield m }
+      else
+        migrations
+      end
+
+      ClickhouseActiverecord::Migrator.new(:up, selected_migrations, schema_migration, target_version).migrate
+    end
+
     def down(target_version = nil)
       selected_migrations = if block_given?
         migrations.select { |m| yield m }

data/lib/clickhouse-activerecord/tasks.rb

@@ -14,7 +14,7 @@ module ClickhouseActiverecord
       connection.create_database @configuration["database"]
     rescue ActiveRecord::StatementInvalid => e
       if e.cause.to_s.include?('already exists')
-        raise ActiveRecord::Tasks::DatabaseAlreadyExists
+        raise ActiveRecord::DatabaseAlreadyExists
       else
         raise
       end
@@ -43,7 +43,15 @@ module ClickhouseActiverecord
     end
 
     def structure_load(*args)
-      File.read(args.first).split(";\n\n").each { |sql| connection.execute(sql) }
+      File.read(args.first).split(";\n\n").each do |sql|
+        if sql.gsub(/[a-z]/i, '').blank?
+          next
+        elsif sql =~ /^INSERT INTO/
+          connection.do_execute(sql, nil, format: nil)
+        else
+          connection.execute(sql)
+        end
+      end
     end
 
     def migrate

data/lib/clickhouse-activerecord/version.rb

@@ -1,3 +1,3 @@
 module ClickhouseActiverecord
-  VERSION = '0.5.2'
+  VERSION = '0.5.8'
 end

data/lib/clickhouse-activerecord.rb

@@ -2,6 +2,9 @@
 
 require 'active_record/connection_adapters/clickhouse_adapter'
 
+require_relative '../core_extensions/active_record/migration/command_recorder'
+ActiveRecord::Migration::CommandRecorder.include CoreExtensions::ActiveRecord::Migration::CommandRecorder
+
 if defined?(Rails::Railtie)
   require 'clickhouse-activerecord/railtie'
   require 'clickhouse-activerecord/schema'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clickhouse-activerecord
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.8
 platform: ruby
 authors:
 - Sergey Odintsov
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-08-04 00:00:00.000000000 Z
+date: 2022-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -42,14 +42,14 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.15'
 - !ruby/object:Gem::Dependency
@@ -112,6 +112,7 @@ files:
 - bin/console
 - bin/setup
 - clickhouse-activerecord.gemspec
+- core_extensions/active_record/migration/command_recorder.rb
 - lib/active_record/connection_adapters/clickhouse/oid/array.rb
 - lib/active_record/connection_adapters/clickhouse/oid/big_integer.rb
 - lib/active_record/connection_adapters/clickhouse/oid/date.rb
@@ -135,7 +136,7 @@ homepage: https://github.com/pnixx/clickhouse-activerecord
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -150,8 +151,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
-signing_key:
+rubygems_version: 3.0.3
+signing_key: 
 specification_version: 4
 summary: ClickHouse ActiveRecord
 test_files: []