clickhouse-activerecord 1.0.4 → 1.0.6

data/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb

@@ -94,6 +94,23 @@ module ActiveRecord
           args.each { |name| column(name, kind, **options.except(:limit)) }
         end
       end
+
+      class IndexDefinition
+        attr_reader :table, :name, :expression, :type, :granularity, :first, :after, :if_exists, :if_not_exists
+
+        def initialize(table, name, expression, type, granularity, first:, after:, if_exists:, if_not_exists:)
+          @table = table
+          @name = name
+          @expression = expression
+          @type = type
+          @granularity = granularity
+          @first = first
+          @after = after
+          @if_exists = if_exists
+          @if_not_exists = if_not_exists
+        end
+
+      end
     end
   end
 end

data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb

@@ -6,6 +6,8 @@ module ActiveRecord
   module ConnectionAdapters
     module Clickhouse
       module SchemaStatements
+        DEFAULT_RESPONSE_FORMAT = 'JSONCompactEachRowWithNamesAndTypes'.freeze
+
         def execute(sql, name = nil, settings: {})
           do_execute(sql, name, settings: settings)
         end
@@ -33,13 +35,20 @@ module ActiveRecord
         # @link https://clickhouse.com/docs/en/sql-reference/statements/alter/update
         def exec_update(_sql, _name = nil, _binds = [])
           do_execute(_sql, _name, format: nil)
-          true
+          0
         end
 
         # @link https://clickhouse.com/docs/en/sql-reference/statements/delete
         def exec_delete(_sql, _name = nil, _binds = [])
-          do_execute(_sql, _name, format: nil)
-          true
+          log(_sql, "#{adapter_name} #{_name}") do
+            res = request(_sql)
+            begin
+              data = JSON.parse(res.header['x-clickhouse-summary'])
+              data['result_rows'].to_i
+            rescue JSONError
+              0
+            end
+          end
         end
 
         def tables(name = nil)
@@ -48,6 +57,16 @@ module ActiveRecord
           result['data'].flatten
         end
 
+        def functions
+          result = do_system_execute("SELECT name FROM system.functions WHERE origin = 'SQLUserDefined'")
+          return [] if result.nil?
+          result['data'].flatten
+        end
+
+        def show_create_function(function)
+          do_execute("SELECT create_query FROM system.functions WHERE origin = 'SQLUserDefined' AND name = '#{function}'", format: nil)
+        end
+
         def table_options(table)
           sql = show_create_table(table)
           { options: sql.gsub(/^(?:.*?)(?:ENGINE = (.*?))?( AS SELECT .*?)?$/, '\\1').presence, as: sql.match(/^CREATE (?:.*?) AS (SELECT .*?)$/).try(:[], 1) }.compact
@@ -58,25 +77,29 @@ module ActiveRecord
           []
         end
 
+        def add_index_options(table_name, expression, **options)
+          options.assert_valid_keys(:name, :type, :granularity, :first, :after, :if_not_exists, :if_exists)
+
+          validate_index_length!(table_name, options[:name])
+
+          IndexDefinition.new(table_name, options[:name], expression, options[:type], options[:granularity], first: options[:first], after: options[:after], if_not_exists: options[:if_not_exists], if_exists: options[:if_exists])
+        end
+
         def data_sources
           tables
         end
 
         def do_system_execute(sql, name = nil)
           log_with_debug(sql, "#{adapter_name} #{name}") do
-            res = @connection.post("/?#{@connection_config.to_param}", "#{sql} FORMAT JSONCompact", 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
-
-            process_response(res)
+            res = request(sql, DEFAULT_RESPONSE_FORMAT)
+            process_response(res, DEFAULT_RESPONSE_FORMAT, sql)
           end
         end
 
-        def do_execute(sql, name = nil, format: 'JSONCompact', settings: {})
+        def do_execute(sql, name = nil, format: DEFAULT_RESPONSE_FORMAT, settings: {})
           log(sql, "#{adapter_name} #{name}") do
-            formatted_sql = apply_format(sql, format)
-            request_params = @connection_config || {}
-            res = @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
-
-            process_response(res)
+            res = request(sql, format, settings)
+            process_response(res, format, sql)
           end
         end
 
@@ -96,7 +119,7 @@ module ActiveRecord
             if (duplicate = inserting.detect { |v| inserting.count(v) > 1 })
               raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict."
             end
-            do_execute(insert_versions_sql(inserting), nil, settings: {max_partitions_per_insert_block: [100, inserting.size].max})
+            do_execute(insert_versions_sql(inserting), nil, format: nil, settings: {max_partitions_per_insert_block: [100, inserting.size].max})
           end
         end
 
@@ -112,17 +135,28 @@ module ActiveRecord
 
         private
 
+        # Make HTTP request to ClickHouse server
+        # @param [String] sql
+        # @param [String, nil] format
+        # @param [Hash] settings
+        # @return [Net::HTTPResponse]
+        def request(sql, format = nil, settings = {})
+          formatted_sql = apply_format(sql, format)
+          request_params = @connection_config || {}
+          @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
+        end
+
         def apply_format(sql, format)
           format ? "#{sql} FORMAT #{format}" : sql
         end
 
-        def process_response(res)
+        def process_response(res, format, sql = nil)
           case res.code.to_i
           when 200
             if res.body.to_s.include?("DB::Exception")
-              raise ActiveRecord::ActiveRecordError, "Response code: #{res.code}:\n#{res.body}"
+              raise ActiveRecord::ActiveRecordError, "Response code: #{res.code}:\n#{res.body}#{sql ? "\nQuery: #{sql}" : ''}"
             else
-              res.body.presence && JSON.parse(res.body)
+              format_body_response(res.body, format)
             end
           else
             case res.body
@@ -168,8 +202,7 @@ module ActiveRecord
 
           return data unless data.empty?
 
-          raise ActiveRecord::StatementInvalid,
-            "Could not find table '#{table_name}'"
+          raise ActiveRecord::StatementInvalid, "Could not find table '#{table_name}'"
         end
         alias column_definitions table_structure
 
@@ -203,6 +236,44 @@ module ActiveRecord
         def has_default_function?(default_value, default) # :nodoc:
           !default_value && (%r{\w+\(.*\)} === default)
         end
+
+        def format_body_response(body, format)
+          return body if body.blank?
+
+          case format
+          when 'JSONCompact'
+            format_from_json_compact(body)
+          when 'JSONCompactEachRowWithNamesAndTypes'
+            format_from_json_compact_each_row_with_names_and_types(body)
+          else
+            body
+          end
+        end
+
+        def format_from_json_compact(body)
+          parse_json_payload(body)
+        end
+
+        def format_from_json_compact_each_row_with_names_and_types(body)
+          rows = body.split("\n").map { |row| parse_json_payload(row) }
+          names, types, *data = rows
+
+          meta = names.zip(types).map do |name, type|
+            {
+              'name' => name,
+              'type' => type
+            }
+          end
+
+          {
+            'meta' => meta,
+            'data' => data
+          }
+        end
+
+        def parse_json_payload(payload)
+          JSON.parse(payload, decimal_class: BigDecimal)
+        end
       end
     end
   end

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -73,10 +73,11 @@ module ActiveRecord
       def is_view=(value)
         @is_view = value
       end
-      #
-      # def arel_table # :nodoc:
-      #   @arel_table ||= Arel::Table.new(table_name, type_caster: type_caster)
-      # end
+
+      def _delete_record(constraints)
+        raise ActiveRecord::ActiveRecordError.new('Deleting a row is not possible without a primary key') unless self.primary_key
+        super
+      end
     end
   end
 
@@ -148,6 +149,10 @@ module ActiveRecord
         !native_database_types[type].nil?
       end
 
+      def supports_indexes_in_create?
+        true
+      end
+
       class << self
         def extract_limit(sql_type) # :nodoc:
           case sql_type
@@ -267,7 +272,7 @@ module ActiveRecord
         sql = apply_cluster "CREATE DATABASE #{quote_table_name(name)}"
         log_with_debug(sql, adapter_name) do
           res = @connection.post("/?#{@connection_config.except(:database).to_param}", sql)
-          process_response(res)
+          process_response(res, DEFAULT_RESPONSE_FORMAT)
         end
       end
 
@@ -307,12 +312,23 @@ module ActiveRecord
         end
       end
 
+      def create_function(name, body)
+        fd = "CREATE FUNCTION #{apply_cluster(quote_table_name(name))} AS #{body}"
+        do_execute(fd, format: nil)
+      end
+
       # Drops a ClickHouse database.
       def drop_database(name) #:nodoc:
         sql = apply_cluster "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
         log_with_debug(sql, adapter_name) do
           res = @connection.post("/?#{@connection_config.except(:database).to_param}", sql)
-          process_response(res)
+          process_response(res, DEFAULT_RESPONSE_FORMAT)
+        end
+      end
+
+      def drop_functions
+        functions.each do |function|
+          drop_function(function)
         end
       end
 
@@ -335,6 +351,16 @@ module ActiveRecord
         end
       end
 
+      def drop_function(name, options = {})
+        query = "DROP FUNCTION"
+        query = "#{query} IF EXISTS " if options[:if_exists]
+        query = "#{query} #{quote_table_name(name)}"
+        query = apply_cluster(query)
+        query = "#{query} SYNC" if options[:sync]
+
+        do_execute(query, format: nil)
+      end
+
       def add_column(table_name, column_name, type, **options)
         return if options[:if_not_exists] == true && column_exists?(table_name, column_name, type)
 
@@ -349,8 +375,8 @@ module ActiveRecord
         execute("ALTER TABLE #{quote_table_name(table_name)} #{remove_column_for_alter(table_name, column_name, type, **options)}", nil, settings: {wait_end_of_query: 1, send_progress_in_http_headers: 1})
       end
 
-      def change_column(table_name, column_name, type, options = {})
-        result = do_execute("ALTER TABLE #{quote_table_name(table_name)} #{change_column_for_alter(table_name, column_name, type, options)}", nil, settings: {wait_end_of_query: 1, send_progress_in_http_headers: 1})
+      def change_column(table_name, column_name, type, **options)
+        result = do_execute("ALTER TABLE #{quote_table_name(table_name)} #{change_column_for_alter(table_name, column_name, type, **options)}", nil, settings: {wait_end_of_query: 1, send_progress_in_http_headers: 1})
         raise "Error parse json response: #{result}" if result.presence && !result.is_a?(Hash)
       end
 
@@ -364,6 +390,39 @@ module ActiveRecord
         change_column table_name, column_name, nil, {default: default}.compact
       end
 
+      # Adds index description to tables metadata
+      # @link https://clickhouse.com/docs/en/sql-reference/statements/alter/skipping-index
+      def add_index(table_name, expression, **options)
+        index = add_index_options(apply_cluster(table_name), expression, **options)
+        execute schema_creation.accept(CreateIndexDefinition.new(index))
+      end
+
+      # Removes index description from tables metadata and deletes index files from disk
+      def remove_index(table_name, name)
+        query = apply_cluster("ALTER TABLE #{quote_table_name(table_name)}")
+        execute "#{query} DROP INDEX #{quote_column_name(name)}"
+      end
+
+      # Rebuilds the secondary index name for the specified partition_name
+      def rebuild_index(table_name, name, if_exists: false, partition: nil)
+        query = [apply_cluster("ALTER TABLE #{quote_table_name(table_name)}")]
+        query << 'MATERIALIZE INDEX'
+        query << 'IF EXISTS' if if_exists
+        query << quote_column_name(name)
+        query << "IN PARTITION #{quote_column_name(partition)}" if partition
+        execute query.join(' ')
+      end
+
+      # Deletes the secondary index files from disk without removing description
+      def clear_index(table_name, name, if_exists: false, partition: nil)
+        query = [apply_cluster("ALTER TABLE #{quote_table_name(table_name)}")]
+        query << 'CLEAR INDEX'
+        query << 'IF EXISTS' if if_exists
+        query << quote_column_name(name)
+        query << "IN PARTITION #{quote_column_name(partition)}" if partition
+        execute query.join(' ')
+      end
+
       def cluster
         @config[:cluster_name]
       end
@@ -419,9 +478,9 @@ module ActiveRecord
         result
       end
 
-      def change_column_for_alter(table_name, column_name, type, options = {})
+      def change_column_for_alter(table_name, column_name, type, **options)
         td = create_table_definition(table_name)
-        cd = td.new_column_definition(column_name, type, options)
+        cd = td.new_column_definition(column_name, type, **options)
         schema_creation.accept(ChangeColumnDefinition.new(cd, column_name))
       end
 

data/lib/arel/visitors/clickhouse.rb

@@ -13,6 +13,13 @@ module Arel
         end
       end
 
+      # https://clickhouse.com/docs/en/sql-reference/statements/delete
+      # DELETE and UPDATE in ClickHouse working only without table name
+      def visit_Arel_Attributes_Attribute(o, collector)
+        collector << quote_table_name(o.relation.table_alias || o.relation.name) << '.' unless collector.value.start_with?('DELETE FROM ') || collector.value.include?(' UPDATE ')
+        collector << quote_column_name(o.name)
+      end
+
       def visit_Arel_Nodes_SelectOptions(o, collector)
         maybe_visit o.settings, super
       end

data/lib/clickhouse-activerecord/schema_dumper.rb

@@ -34,8 +34,12 @@ HEADER
     end
 
     def tables(stream)
-      sorted_tables = @connection.tables.sort {|a,b| @connection.show_create_table(a).match(/^CREATE\s+(MATERIALIZED\s+)?VIEW/) ? 1 : a <=> b }
+      functions = @connection.functions
+      functions.each do |function|
+        function(function, stream)
+      end
 
+      sorted_tables = @connection.tables.sort {|a,b| @connection.show_create_table(a).match(/^CREATE\s+(MATERIALIZED\s+)?VIEW/) ? 1 : a <=> b }
       sorted_tables.each do |table_name|
         table(table_name, stream) unless ignored?(table_name)
       end
@@ -104,7 +108,13 @@ HEADER
             end
           end
 
-          indexes_in_create(table, tbl)
+          indexes = sql.scan(/INDEX \S+ \S+ TYPE .*? GRANULARITY \d+/)
+          if indexes.any?
+            tbl.puts ''
+            indexes.flatten.map!(&:strip).each do |index|
+              tbl.puts "    t.index #{index_parts(index).join(', ')}"
+            end
+          end
 
           tbl.puts "  end"
           tbl.puts
@@ -119,6 +129,13 @@ HEADER
       end
     end
 
+    def function(function, stream)
+      stream.puts "  # FUNCTION: #{function}"
+      sql = @connection.show_create_function(function)
+      stream.puts "  # SQL: #{sql}" if sql
+      stream.puts "  create_function \"#{function}\", \"#{sql.gsub(/^CREATE FUNCTION (.*?) AS/, '').strip}\"" if sql
+    end
+
     def format_options(options)
       if options && options[:options]
         options[:options] = options[:options].gsub(/^Replicated(.*?)\('[^']+',\s*'[^']+',?\s?([^\)]*)?\)/, "\\1(\\2)")
@@ -154,5 +171,16 @@ HEADER
       spec[:array] = schema_array(column)
       spec.merge(super).compact
     end
+
+    def index_parts(index)
+      idx = index.match(/^INDEX (?<name>\S+) (?<expr>.*?) TYPE (?<type>.*?) GRANULARITY (?<granularity>\d+)$/)
+      index_parts = [
+        format_index_parts(idx['expr']),
+        "name: #{format_index_parts(idx['name'])}",
+        "type: #{format_index_parts(idx['type'])}",
+      ]
+      index_parts << "granularity: #{idx['granularity']}" if idx['granularity']
+      index_parts
+    end
   end
 end

data/lib/clickhouse-activerecord/tasks.rb

@@ -5,12 +5,12 @@ module ClickhouseActiverecord
     delegate :connection, :establish_connection, to: ActiveRecord::Base
 
     def initialize(configuration)
-      @configuration = configuration.with_indifferent_access
+      @configuration = configuration
     end
 
     def create
       establish_master_connection
-      connection.create_database @configuration['database']
+      connection.create_database @configuration.database
     rescue ActiveRecord::StatementInvalid => e
       if e.cause.to_s.include?('already exists')
         raise ActiveRecord::DatabaseAlreadyExists
@@ -21,7 +21,7 @@ module ClickhouseActiverecord
 
     def drop
       establish_master_connection
-      connection.drop_database @configuration['database']
+      connection.drop_database @configuration.database
     end
 
     def purge
@@ -31,12 +31,28 @@ module ClickhouseActiverecord
     end
 
     def structure_dump(*args)
-      tables = connection.execute("SHOW TABLES FROM #{@configuration['database']}")['data'].flatten
+      establish_master_connection
+
+      # get all tables
+      tables = connection.execute("SHOW TABLES FROM #{@configuration.database} WHERE name NOT LIKE '.inner_id.%'")['data'].flatten.map do |table|
+        next if %w[schema_migrations ar_internal_metadata].include?(table)
+        connection.show_create_table(table).gsub("#{@configuration.database}.", '')
+      end.compact
+
+      # sort view to last
+      tables.sort_by! {|table| table.match(/^CREATE\s+(MATERIALIZED\s+)?VIEW/) ? 1 : 0}
 
+      # get all functions
+      functions = connection.execute("SELECT create_query FROM system.functions WHERE origin = 'SQLUserDefined'")['data'].flatten
+
+      # put to file
       File.open(args.first, 'w:utf-8') do |file|
+        functions.each do |function|
+          file.puts function + ";\n\n"
+        end
+        
         tables.each do |table|
-          next if table.match(/\.inner/)
-          file.puts connection.execute("SHOW CREATE TABLE #{table}")['data'].try(:first).try(:first).gsub("#{@configuration['database']}.", '') + ";\n\n"
+          file.puts table + ";\n\n"
         end
       end
     end

data/lib/clickhouse-activerecord/version.rb

@@ -1,3 +1,3 @@
 module ClickhouseActiverecord
-  VERSION = '1.0.4'
+  VERSION = '1.0.6'
 end

data/lib/clickhouse-activerecord.rb

@@ -5,15 +5,12 @@ require 'active_record/connection_adapters/clickhouse_adapter'
 require 'core_extensions/active_record/internal_metadata'
 require 'core_extensions/active_record/relation'
 require 'core_extensions/active_record/schema_migration'
-
+require 'core_extensions/active_record/migration/command_recorder'
 require 'core_extensions/arel/nodes/select_core'
 require 'core_extensions/arel/nodes/select_statement'
 require 'core_extensions/arel/select_manager'
 require 'core_extensions/arel/table'
 
-require_relative '../core_extensions/active_record/migration/command_recorder'
-ActiveRecord::Migration::CommandRecorder.include CoreExtensions::ActiveRecord::Migration::CommandRecorder
-
 if defined?(Rails::Railtie)
   require 'clickhouse-activerecord/railtie'
   require 'clickhouse-activerecord/schema'
@@ -24,9 +21,10 @@ end
 
 module ClickhouseActiverecord
   def self.load
-    ActiveRecord::InternalMetadata.singleton_class.prepend(CoreExtensions::ActiveRecord::InternalMetadata::ClassMethods)
+    ActiveRecord::InternalMetadata.prepend(CoreExtensions::ActiveRecord::InternalMetadata)
+    ActiveRecord::Migration::CommandRecorder.include(CoreExtensions::ActiveRecord::Migration::CommandRecorder)
     ActiveRecord::Relation.prepend(CoreExtensions::ActiveRecord::Relation)
-    ActiveRecord::SchemaMigration.singleton_class.prepend(CoreExtensions::ActiveRecord::SchemaMigration::ClassMethods)
+    ActiveRecord::SchemaMigration.prepend(CoreExtensions::ActiveRecord::SchemaMigration)
 
     Arel::Nodes::SelectCore.prepend(CoreExtensions::Arel::Nodes::SelectCore)
     Arel::Nodes::SelectStatement.prepend(CoreExtensions::Arel::Nodes::SelectStatement)

data/lib/core_extensions/active_record/internal_metadata.rb

@@ -1,46 +1,54 @@
 module CoreExtensions
   module ActiveRecord
     module InternalMetadata
-      module ClassMethods
 
-        def []=(key, value)
-          row = final.find_by(key: key)
-          if row.nil? || row.value != value
-            create!(key: key, value: value)
-          end
-        end
+      def create_table
+        return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
+        return if table_exists? || !enabled?
+
+        key_options = connection.internal_string_options_for_primary_key
+        table_options = {
+          id: false,
+          options: 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key',
+          if_not_exists: true
+        }
+        full_config = connection.instance_variable_get(:@config) || {}
+
+        if full_config[:distributed_service_tables]
+          table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(created_at)')
 
-        def [](key)
-          final.where(key: key).pluck(:value).first
+          distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}"
+        else
+          distributed_suffix = ''
         end
 
-        def create_table
-          return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
-          return if table_exists? || !enabled?
-
-          key_options = connection.internal_string_options_for_primary_key
-          table_options = {
-            id: false,
-            options: connection.adapter_name.downcase == 'clickhouse' ? 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key' : '',
-            if_not_exists: true
-          }
-          full_config = connection.instance_variable_get(:@config) || {}
-
-          if full_config[:distributed_service_tables]
-            table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(created_at)')
-
-            distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}"
-          else
-            distributed_suffix = ''
-          end
-
-          connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t|
-            t.string :key, **key_options
-            t.string :value
-            t.timestamps
-          end
+        connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t|
+          t.string :key, **key_options
+          t.string :value
+          t.timestamps
         end
       end
+
+      private
+
+      def update_entry(key, new_value)
+        return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
+
+        create_entry(key, new_value)
+      end
+
+      def select_entry(key)
+        return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
+
+        sm = ::Arel::SelectManager.new(arel_table)
+        sm.final! if connection.table_options(table_name)[:options] =~ /^ReplacingMergeTree/
+        sm.project(::Arel.star)
+        sm.where(arel_table[primary_key].eq(::Arel::Nodes::BindParam.new(key)))
+        sm.order(arel_table[primary_key].asc)
+        sm.limit = 1
+
+        connection.select_one(sm, "#{self.class} Load")
+      end
     end
   end
 end