clickhouse-activerecord 1.3.1 → 1.5.0

data/lib/active_record/connection_adapters/clickhouse/statement/format_manager.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Clickhouse
+      class Statement
+        class FormatManager
+
+          def initialize(sql, format:)
+            @sql = sql.strip
+            @format = format
+          end
+
+          def apply
+            return @sql if skip_format? || @format.blank?
+
+            "#{@sql} FORMAT #{@format}"
+          end
+
+          def skip_format?
+            system_command? || schema_command? || format_specified? || delete?
+          end
+
+          private
+
+          def system_command?
+            /\Asystem|\Aoptimize/i.match?(@sql)
+          end
+
+          def schema_command?
+            /\Acreate|\Aalter|\Adrop|\Arename/i.match?(@sql)
+          end
+
+          def format_specified?
+            /format [a-z]+\z/i.match?(@sql)
+          end
+
+          def delete?
+            /\Adelete from/i.match?(@sql)
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse/statement/response_processor.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Clickhouse
+      class Statement
+        class ResponseProcessor
+
+          DB_EXCEPTION_REGEXP = /\ACode:\s+\d+\.\s+DB::Exception:/.freeze
+
+          def initialize(raw_response, format, sql)
+            @raw_response = raw_response
+            @body = raw_response.body
+            @format = format
+            @sql = sql
+          end
+
+          def process
+            if success?
+              process_successful_response
+            else
+              raise_database_error!
+            end
+          rescue JSON::ParserError
+            @body
+          end
+
+          private
+
+          def success?
+            @raw_response.code.to_i == 200
+          end
+
+          def process_successful_response
+            raise_generic!(@sql) if @body.include?('DB::Exception') && @body.match?(DB_EXCEPTION_REGEXP)
+
+            format_body_response
+          end
+
+          def raise_generic!(sql = nil)
+            raise ActiveRecord::ActiveRecordError, "Response code: #{@raw_response.code}:\n#{@body}#{"\nQuery: #{sql}" if sql}"
+          end
+
+          def format_body_response
+            return @body if @body.blank?
+
+            case @format
+            when 'JSONCompact'
+              format_from_json_compact(@body)
+            when 'JSONCompactEachRowWithNamesAndTypes'
+              format_from_json_compact_each_row_with_names_and_types(@body)
+            else
+              @body
+            end
+          rescue JSON::ParserError
+            @body
+          end
+
+          def format_from_json_compact(body)
+            parse_json_payload(body)
+          end
+
+          def format_from_json_compact_each_row_with_names_and_types(body)
+            rows = body.each_line.map { |row| parse_json_payload(row) }
+            names, types, *data = rows
+
+            meta = names.zip(types).map do |name, type|
+              {
+                'name' => name,
+                'type' => type
+              }
+            end
+
+            {
+              'meta' => meta,
+              'data' => data
+            }
+          end
+
+          def parse_json_payload(payload)
+            JSON.parse(payload, decimal_class: BigDecimal)
+          end
+
+          def raise_database_error!
+            case @body
+            when /DB::Exception:.*\(UNKNOWN_DATABASE\)/
+              raise ActiveRecord::NoDatabaseError
+            when /DB::Exception:.*\(DATABASE_ALREADY_EXISTS\)/
+              raise ActiveRecord::DatabaseAlreadyExists
+            else
+              raise_generic!
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse/statement.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'active_record/connection_adapters/clickhouse/statement/format_manager'
+require 'active_record/connection_adapters/clickhouse/statement/response_processor'
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Clickhouse
+      class Statement
+
+        attr_reader :format
+        attr_writer :response
+
+        def initialize(sql, format:)
+          @sql = sql
+          @format = format
+        end
+
+        def formatted_sql
+          @formatted_sql ||= FormatManager.new(@sql, format: @format).apply
+        end
+
+        def processed_response
+          ResponseProcessor.new(@response, @format, @sql).process
+        end
+
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -16,6 +16,7 @@ require 'active_record/connection_adapters/clickhouse/column'
 require 'active_record/connection_adapters/clickhouse/quoting'
 require 'active_record/connection_adapters/clickhouse/schema_creation'
 require 'active_record/connection_adapters/clickhouse/schema_statements'
+require 'active_record/connection_adapters/clickhouse/statement'
 require 'active_record/connection_adapters/clickhouse/table_definition'
 require 'net/http'
 require 'openssl'
@@ -82,6 +83,8 @@ module ActiveRecord
       include Clickhouse::Quoting
 
       ADAPTER_NAME = 'Clickhouse'.freeze
+      DEFAULT_RESPONSE_FORMAT = 'JSONCompactEachRowWithNamesAndTypes'.freeze
+      USER_AGENT = "ClickHouse ActiveRecord #{ClickhouseActiverecord::VERSION}"
       NATIVE_DATABASE_TYPES = {
         string: { name: 'String' },
         integer: { name: 'UInt32' },
@@ -137,6 +140,7 @@ module ActiveRecord
 
         @connection_config = { user: @config[:username], password: @config[:password], database: @config[:database] }.compact
         @debug = @config[:debug] || false
+        @response_format = @config[:format] || DEFAULT_RESPONSE_FORMAT
 
         @prepared_statements = false
 
@@ -145,7 +149,7 @@ module ActiveRecord
 
       # Return ClickHouse server version
       def server_version
-        @server_version ||= do_system_execute('SELECT version()')['data'][0][0]
+        @server_version ||= select_value('SELECT version()')
       end
 
       # Savepoints are not supported, noop
@@ -310,10 +314,7 @@ module ActiveRecord
       # Create a new ClickHouse database.
       def create_database(name)
         sql = apply_cluster "CREATE DATABASE #{quote_table_name(name)}"
-        log_with_debug(sql, adapter_name) do
-          res = @connection.post("/?#{@connection_config.except(:database).to_param}", sql)
-          process_response(res, DEFAULT_RESPONSE_FORMAT)
-        end
+        do_system_execute sql, adapter_name, except_params: [:database]
       end
 
       def create_view(table_name, request_settings: {}, **options)
@@ -326,7 +327,7 @@ module ActiveRecord
           drop_table(table_name, options.merge(if_exists: true))
         end
 
-        do_execute(schema_creation.accept(td), format: nil, settings: request_settings)
+        execute(schema_creation.accept(td), settings: request_settings)
       end
 
       def create_table(table_name, request_settings: {}, **options, &block)
@@ -343,7 +344,7 @@ module ActiveRecord
           drop_table(table_name, options.merge(if_exists: true))
         end
 
-        do_execute(schema_creation.accept(td), format: nil, settings: request_settings)
+        execute(schema_creation.accept(td), settings: request_settings)
 
         if options[:with_distributed]
           distributed_table_name = options.delete(:with_distributed)
@@ -358,16 +359,13 @@ module ActiveRecord
 
       def create_function(name, body, **options)
         fd = "CREATE#{' OR REPLACE' if options[:force]} FUNCTION #{apply_cluster(quote_table_name(name))} AS #{body}"
-        do_execute(fd, format: nil)
+        execute(fd)
       end
 
       # Drops a ClickHouse database.
       def drop_database(name) #:nodoc:
         sql = apply_cluster "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
-        log_with_debug(sql, adapter_name) do
-          res = @connection.post("/?#{@connection_config.except(:database).to_param}", sql)
-          process_response(res, DEFAULT_RESPONSE_FORMAT)
-        end
+        do_system_execute sql, adapter_name, except_params: [:database]
       end
 
       def drop_functions
@@ -377,7 +375,7 @@ module ActiveRecord
       end
 
       def rename_table(table_name, new_name)
-        do_execute apply_cluster "RENAME TABLE #{quote_table_name(table_name)} TO #{quote_table_name(new_name)}"
+        execute apply_cluster "RENAME TABLE #{quote_table_name(table_name)} TO #{quote_table_name(new_name)}"
       end
 
       def drop_table(table_name, options = {}) # :nodoc:
@@ -387,7 +385,7 @@ module ActiveRecord
         query = apply_cluster(query)
         query = "#{query} SYNC" if options[:sync]
 
-        do_execute(query)
+        execute(query)
 
         if options[:with_distributed]
           distributed_table_name = options.delete(:with_distributed)
@@ -402,25 +400,19 @@ module ActiveRecord
         query = apply_cluster(query)
         query = "#{query} SYNC" if options[:sync]
 
-        do_execute(query, format: nil)
+        execute(query)
       end
 
       def add_column(table_name, column_name, type, **options)
-        return if options[:if_not_exists] == true && column_exists?(table_name, column_name, type)
-
-        at = create_alter_table table_name
-        at.add_column(column_name, type, **options)
-        execute(schema_creation.accept(at), nil, settings: {wait_end_of_query: 1, send_progress_in_http_headers: 1})
+        with_settings(wait_end_of_query: 1, send_progress_in_http_headers: 1) { super }
       end
 
       def remove_column(table_name, column_name, type = nil, **options)
-        return if options[:if_exists] == true && !column_exists?(table_name, column_name)
-
-        execute("ALTER TABLE #{quote_table_name(table_name)} #{remove_column_for_alter(table_name, column_name, type, **options)}", nil, settings: {wait_end_of_query: 1, send_progress_in_http_headers: 1})
+        with_settings(wait_end_of_query: 1, send_progress_in_http_headers: 1) { super }
       end
 
       def change_column(table_name, column_name, type, **options)
-        result = do_execute("ALTER TABLE #{quote_table_name(table_name)} #{change_column_for_alter(table_name, column_name, type, **options)}", nil, settings: {wait_end_of_query: 1, send_progress_in_http_headers: 1})
+        result = execute("ALTER TABLE #{quote_table_name(table_name)} #{change_column_for_alter(table_name, column_name, type, **options)}", nil, settings: {wait_end_of_query: 1, send_progress_in_http_headers: 1})
         raise "Error parse json response: #{result}" if result.presence && !result.is_a?(Hash)
       end
 
@@ -479,6 +471,13 @@ module ActiveRecord
         @config[:database]
       end
 
+      # Returns the shard name from the configuration.
+      # This is used to identify the shard in replication paths when using both sharding and replication.
+      # Required when you have multiple shards with replication to ensure unique paths for each shard's replication metadata.
+      def shard
+        @config[:shard_name]
+      end
+
       def use_default_replicated_merge_tree_params?
         database_engine_atomic? && @config[:use_default_replicated_merge_tree_params]
       end
@@ -487,8 +486,17 @@ module ActiveRecord
         (replica || use_default_replicated_merge_tree_params?) && cluster
       end
 
+      # Returns the path for replication metadata.
+      # When sharding is enabled (shard_name is set), the path includes the shard identifier
+      # to ensure unique paths for each shard's replication metadata.
+      # Format with sharding: /clickhouse/tables/{cluster}/{shard}/{database}.{table}
+      # Format without sharding: /clickhouse/tables/{cluster}/{database}.{table}
       def replica_path(table)
-        "/clickhouse/tables/#{cluster}/#{@connection_config[:database]}.#{table}"
+        if shard
+          "/clickhouse/tables/#{cluster}/#{shard}/#{@connection_config[:database]}.#{table}"
+        else
+          "/clickhouse/tables/#{cluster}/#{@connection_config[:database]}.#{table}"
+        end
       end
 
       def database_engine_atomic?

data/lib/arel/visitors/clickhouse.rb

@@ -51,8 +51,14 @@ module Arel
       end
 
       def visit_Arel_Nodes_Final(o, collector)
-        visit o.expr, collector
+        visit o.expr.left, collector
         collector << ' FINAL'
+
+        o.expr.right.each do |join|
+          collector << ' '
+          visit join, collector
+        end
+
         collector
       end
 

data/lib/clickhouse-activerecord/schema_dumper.rb

@@ -31,6 +31,8 @@ module ClickhouseActiverecord
 
     def table(table, stream)
       if table.match(/^\.inner/).nil?
+        sql= ""
+        simple ||= ENV['simple'] == 'true'
         unless simple
           stream.puts "  # TABLE: #{table}"
           sql = @connection.show_create_table(table)
@@ -126,7 +128,7 @@ module ClickhouseActiverecord
       sql = @connection.show_create_function(function)
       if sql
         stream.puts "  # SQL: #{sql}"
-        stream.puts "  create_function \"#{function}\", \"#{sql.gsub(/^CREATE FUNCTION (.*?) AS/, '').strip}\", force: true"
+        stream.puts "  create_function \"#{function}\", \"#{sql.sub(/\ACREATE(OR REPLACE)? FUNCTION .*? AS/, '').strip}\", force: true"
         stream.puts
       end
     end

data/lib/clickhouse-activerecord/tasks.rb

@@ -39,7 +39,6 @@ module ClickhouseActiverecord
 
       # get all tables
       tables = connection.execute("SHOW TABLES FROM #{@configuration.database} WHERE name NOT LIKE '.inner_id.%'")['data'].flatten.map do |table|
-        next if %w[schema_migrations ar_internal_metadata].include?(table)
         connection.show_create_table(table, single_line: false).gsub("#{@configuration.database}.", '')
       end.compact
 
@@ -66,9 +65,9 @@ module ClickhouseActiverecord
         if sql.gsub(/[a-z]/i, '').blank?
           next
         elsif sql =~ /^INSERT INTO/
-          connection.do_execute(sql, nil, format: nil)
+          connection.execute(sql, nil, format: nil)
         elsif sql =~ /^CREATE .*?FUNCTION/
-          connection.do_execute(sql, nil, format: nil)
+          connection.execute(sql, nil, format: nil)
         else
           connection.execute(sql)
         end

data/lib/clickhouse-activerecord/version.rb

@@ -1,3 +1,3 @@
 module ClickhouseActiverecord
-  VERSION = '1.3.1'
+  VERSION = '1.5.0'
 end

data/lib/core_extensions/active_record/relation.rb

@@ -1,6 +1,11 @@
 module CoreExtensions
   module ActiveRecord
     module Relation
+
+      def self.prepended(base)
+        base::VALID_UNSCOPING_VALUES << :final << :settings
+      end
+
       def reverse_order!
         return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
 
@@ -19,17 +24,39 @@ module CoreExtensions
       #
       # An <tt>ActiveRecord::ActiveRecordError</tt> will be raised if database not ClickHouse.
       # @param [Hash] opts
+
+
+      # Specify settings to be used for this single query.
+      # For example:
+      #
+      #   users = User.settings(use_skip_indexes: true).where(name: 'John')
+      #   # SELECT "users".* FROM "users"
+      #   # WHERE "users"."name" = 'John'
+      #   # SETTINGS use_skip_indexes = 1
       def settings(**opts)
         spawn.settings!(**opts)
       end
 
       # @param [Hash] opts
       def settings!(**opts)
-        check_command('SETTINGS')
-        @values[:settings] = (@values[:settings] || {}).merge opts
+        check_command!('SETTINGS')
+        self.settings_values = settings_values.merge opts
         self
       end
 
+      def settings_values
+        @values.fetch(:settings, ::ActiveRecord::QueryMethods::FROZEN_EMPTY_HASH)
+      end
+
+      def settings_values=(value)
+        if ::ActiveRecord::version >= Gem::Version.new('7.2')
+          assert_modifiable!
+        else
+          assert_mutability!
+        end
+        @values[:settings] = value
+      end
+
       # When FINAL is specified, ClickHouse fully merges the data before returning the result and thus performs all data transformations that happen during merges for the given table engine.
       # For example:
       #
@@ -37,16 +64,32 @@ module CoreExtensions
       #   # SELECT users.* FROM users FINAL
       #
       # An <tt>ActiveRecord::ActiveRecordError</tt> will be raised if database not ClickHouse.
-      def final
-        spawn.final!
+      #
+      # @param [Boolean] final
+      def final(final = true)
+        spawn.final!(final)
       end
 
-      def final!
-        check_command('FINAL')
-        @values[:final] = true
+      # @param [Boolean] final
+      def final!(final = true)
+        check_command!('FINAL')
+        self.final_value = final
         self
       end
 
+      def final_value=(value)
+        if ::ActiveRecord::version >= Gem::Version.new('7.2')
+          assert_modifiable!
+        else
+          assert_mutability!
+        end
+        @values[:final] = value
+      end
+
+      def final_value
+        @values.fetch(:final, nil)
+      end
+
       # GROUPING SETS allows you to specify multiple groupings in the GROUP BY clause.
       # Whereas GROUP BY CUBE generates all possible groupings, GROUP BY GROUPING SETS generates only the specified groupings.
       # For example:
@@ -128,20 +171,22 @@ module CoreExtensions
 
       private
 
-      def check_command(cmd)
+      def check_command!(cmd)
         raise ::ActiveRecord::ActiveRecordError, cmd + ' is a ClickHouse specific query clause' unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
       end
 
       def build_arel(connection_or_aliases = nil, aliases = nil)
-        if ::ActiveRecord::version >= Gem::Version.new('7.2')
+        requirement = Gem::Requirement.new('>= 7.2', '< 8.1')
+
+        if requirement.satisfied_by?(::ActiveRecord::version)
           arel = super
         else
           arel = super(connection_or_aliases)
         end
 
-        arel.final! if @values[:final].present?
+        arel.final! if final_value
         arel.limit_by(*@values[:limit_by]) if @values[:limit_by].present?
-        arel.settings(@values[:settings]) if @values[:settings].present?
+        arel.settings(settings_values) unless settings_values.empty?
         arel.using(@values[:using]) if @values[:using].present?
         arel.windows(@values[:windows]) if @values[:windows].present?
 

data/lib/core_extensions/arel/nodes/select_core.rb

@@ -10,6 +10,13 @@ module CoreExtensions
           ::Arel::Nodes::Final.new(super)
         end
 
+        def hash
+          [
+            @source, @set_quantifier, @projections, @optimizer_hints,
+            @wheres, @groups, @havings, @windows, @comment, @final
+          ].hash
+        end
+
         def eql?(other)
           super && final == other.final
         end

data/lib/core_extensions/arel/nodes/select_statement.rb

@@ -10,6 +10,10 @@ module CoreExtensions
           @settings = nil
         end
 
+        def hash
+          [@cores, @orders, @limit, @lock, @offset, @with, @settings].hash
+        end
+
         def eql?(other)
           super && 
             limit_by == other.limit_by &&

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: clickhouse-activerecord
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.5.0
 platform: ruby
 authors:
 - Sergey Odintsov
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-02-13 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -122,6 +121,9 @@ files:
 - lib/active_record/connection_adapters/clickhouse/quoting.rb
 - lib/active_record/connection_adapters/clickhouse/schema_creation.rb
 - lib/active_record/connection_adapters/clickhouse/schema_statements.rb
+- lib/active_record/connection_adapters/clickhouse/statement.rb
+- lib/active_record/connection_adapters/clickhouse/statement/format_manager.rb
+- lib/active_record/connection_adapters/clickhouse/statement/response_processor.rb
 - lib/active_record/connection_adapters/clickhouse/table_definition.rb
 - lib/active_record/connection_adapters/clickhouse_adapter.rb
 - lib/arel/nodes/final.rb
@@ -146,12 +148,10 @@ files:
 - lib/core_extensions/arel/select_manager.rb
 - lib/core_extensions/arel/table.rb
 - lib/generators/clickhouse_migration_generator.rb
-- lib/tasks/clickhouse.rake
 homepage: https://github.com/pnixx/clickhouse-activerecord
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -166,8 +166,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
 summary: ClickHouse ActiveRecord
 test_files: []