clickhouse-activerecord 1.3.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e47b553f60d06d69a8e1f7c70bbc495c32502f0ea75f3ede9c8ac2bafb063ee
-  data.tar.gz: 42ec02c29478ae577fda1f1a726d003e925b2c36faa02fd6a92797a4e05f960c
+  metadata.gz: cd5f6e3d161c6a71abcbafb000b06a3bf0637c07f451bd410c06f9d25fe20901
+  data.tar.gz: e4ba690507249b3a5d9d1a975b5cc3c7d0bfa6018f5744b84f2a382be3bcd996
 SHA512:
-  metadata.gz: 405f20e2c53352ae702b4b462165ccbf5306da2e72733d5c8f80588ba5392637e6b8d2c95d6d61cc9f777d0fc49e48a3578ee6db2fdb292668cb4ec0c52d0471
-  data.tar.gz: 8b3bebc0c3672112c26889555d1eb0b280cbc331758cbf1d5d1733273983e10178929bec76fda6793a654d1ff2c0048b44a58eca533f3ec0a2e8481e7d928314
+  metadata.gz: 3fdc2f20d8540d9599bf6281d19b8408022575d5a6648caaff55337e039bc6538c8da95004f9ab9eea74bd438c514d7907b25cd7ddd58d5f134fbc1ff77732eb
+  data.tar.gz: 45d9da6d58a53e614d2966bfcf0b7db575dce2e8afab6bfdc8cb0f4853ca977a6bfd15e43f767bcce934dbfea371921abc0215f0fdf304661961c58d499bde4a

data/.docker/docker-compose.cluster.yml

@@ -1,5 +1,3 @@
-version: '3.5'
-
 services:
   clickhouse1:
     image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}'

data/.docker/docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3.8'
 services:
   clickhouse:
     image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}'

data/CHANGELOG.md

@@ -1,4 +1,48 @@
-### Version 1.1.2 (Aug 27, 2024)
+### Version 1.4.0 (Sep 18, 2025)
+
+* `CREATE OR REPLACE FUNCTION` in SQL schema dumps in #198
+* Added shard config to handle replica path with shard in #201
+* Add support for simple schema dumping in #203
+* Unscope :final and :settings in #208
+* Encapsulate format logic within Statement and helper classes in #162
+
+### Version 1.3.1 (Feb 12, 2025)
+
+ * Restore replace database from dump schema table creation
+
+ ### Version 1.3.0 (Jan 24, 2025)
+
+ * 🎉 Support for Rails 8.0 #189
+ * Restore multi-line table definitions in structure dumps #187
+ * Use a flag to track updates/deletes in SQL visitor #188
+
+ ### Version 1.2.1 (Nov 18, 2024)
+
+ * Maintain primary key type specificity in #183
+ * Reliably sort functions, views, and materialized views in schema in #181
+ * Improve function dumps in #179
+ * Add support for integer limits in map type in #178
+ * Add support for `request_settings` in create_table
+
+ ### Version 1.2.0 (Oct 23, 2024)
+
+ * Fix for function creation in `structure.sql` #166
+ * Add `group_by_grouping_sets` query method #161
+ * Add support for `CREATE FUNCTION` and `CREATE OR REPLACE FUNCTION`; the later in schema loading #146
+ * Add support for `LIMIT BY` clause #169
+ * Include column definitions in schema dump if the column name is not `id` #173
+ * Add blank line after create_function in schema #170
+ * Improve DB::Exception error handling #164
+ * SchemaDumper adds materialized view destination #159
+ * Add Array support to Map #158
+ * Add support codec compression parameter #135
+
+ ### Version 1.1.3 (Sep 27, 2024)
+
+ * Fix schema dumper #163
+
+ ### Version 1.1.2 (Aug 27, 2024)
+
 * 🎉 Support for rails 7.2 #156
 * Add method `views` for getting table `View` list in #152
 * Add support for Map datatype in #144

data/README.md

@@ -33,6 +33,9 @@ default: &default
   migrations_paths: db/clickhouse # optional, default: db/migrate_clickhouse
   cluster_name: 'cluster_name' # optional for creating tables in cluster 
   replica_name: '{replica}' # replica macros name, optional for creating replicated tables
+  read_timeout: 300 # change network timeouts, by default 60 seconds
+  write_timeout: 300
+  keep_alive_timeout: 300
 ```
 
 Alternatively if you wish to pass a custom `Net::HTTP` transport (or any other

data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb

@@ -6,22 +6,55 @@ module ActiveRecord
   module ConnectionAdapters
     module Clickhouse
       module SchemaStatements
-        DEFAULT_RESPONSE_FORMAT = 'JSONCompactEachRowWithNamesAndTypes'.freeze
 
-        DB_EXCEPTION_REGEXP = /\ACode:\s+\d+\.\s+DB::Exception:/.freeze
+        def with_settings(**settings)
+          @block_settings ||= {}
+          prev_settings = @block_settings
+          @block_settings = @block_settings.merge(settings)
+          yield
+        ensure
+          @block_settings = prev_settings
+        end
 
-        def execute(sql, name = nil, settings: {})
-          do_execute(sql, name, settings: settings)
+        # Request a specific format for the duration of the provided block.
+        # Pass `nil` to explicitly send the SQL statement without a `FORMAT` clause.
+        # @param [String, nil] format
+        #
+        # @example Specify CSVWithNamesAndTypes format
+        #   with_response_format('CSVWithNamesAndTypes') do
+        #     Table.connection.execute('SELECT * FROM table')
+        #   end
+        #   # sends and executes "SELECT * FROM table FORMAT CSVWithNamesAndTypes"
+        #
+        # @example Specify no format
+        #  with_response_format(nil) do
+        #    Table.connection.execute('SELECT * FROM table')
+        #   end
+        #   # sends and executes "SELECT * FROM table"
+        def with_response_format(format)
+          prev_format = @response_format
+          @response_format = format
+          yield
+        ensure
+          @response_format = prev_format
+        end
+
+        def execute(sql, name = nil, format: @response_format, settings: {})
+          with_response_format(format) do
+            log(sql, [adapter_name, name].compact.join(' ')) do
+              raw_execute(sql, settings: settings)
+            end
+          end
         end
 
-        def exec_insert(sql, name, _binds, _pk = nil, _sequence_name = nil, returning: nil)
-          new_sql = sql.dup.sub(/ (DEFAULT )?VALUES/, " VALUES")
-          do_execute(new_sql, name, format: nil)
+        def exec_insert(sql, name = nil, _binds = [], _pk = nil, _sequence_name = nil, returning: nil)
+          new_sql = sql.sub(/ (DEFAULT )?VALUES/, " VALUES")
+          with_response_format(nil) { execute(new_sql, name) }
           true
         end
 
         def internal_exec_query(sql, name = nil, binds = [], prepare: false, async: false, allow_retry: false)
-          result = do_execute(sql, name)
+          result = execute(sql, name)
           columns = result['meta'].map { |m| m['name'] }
           types = {}
           result['meta'].each_with_index do |m, i|
@@ -37,24 +70,25 @@ module ActiveRecord
         end
 
         def exec_insert_all(sql, name)
-          do_execute(sql, name, format: nil)
+          with_response_format(nil) { execute(sql, name) }
           true
         end
 
         # @link https://clickhouse.com/docs/en/sql-reference/statements/alter/update
-        def exec_update(_sql, _name = nil, _binds = [])
-          do_execute(_sql, _name, format: nil)
+        def exec_update(sql, name = nil, _binds = [])
+          execute(sql, name)
           0
         end
 
         # @link https://clickhouse.com/docs/en/sql-reference/statements/delete
-        def exec_delete(_sql, _name = nil, _binds = [])
-          log(_sql, "#{adapter_name} #{_name}") do
-            res = request(_sql)
+        def exec_delete(sql, name = nil, _binds = [])
+          log(sql, "#{adapter_name} #{name}") do
+            statement = Statement.new(sql, format: @response_format)
+            res = request(statement)
             begin
               data = JSON.parse(res.header['x-clickhouse-summary'])
               data['result_rows'].to_i
-            rescue JSONError
+            rescue JSON::ParserError
               0
             end
           end
@@ -85,7 +119,10 @@ module ActiveRecord
         end
 
         def show_create_function(function)
-          do_execute("SELECT create_query FROM system.functions WHERE origin = 'SQLUserDefined' AND name = '#{function}'", format: nil)
+          result = do_system_execute("SELECT create_query FROM system.functions WHERE origin = 'SQLUserDefined' AND name = '#{function}'")
+          return if result.nil?
+
+          result['data'].flatten.first.sub(/\ACREATE FUNCTION/, 'CREATE OR REPLACE FUNCTION')
         end
 
         def table_options(table)
@@ -110,18 +147,18 @@ module ActiveRecord
           tables
         end
 
-        def do_system_execute(sql, name = nil)
-          log_with_debug(sql, "#{adapter_name} #{name}") do
-            res = request(sql, DEFAULT_RESPONSE_FORMAT)
-            process_response(res, DEFAULT_RESPONSE_FORMAT, sql)
+        def do_system_execute(sql, name = nil, except_params: [])
+          log_with_debug(sql, [adapter_name, name].compact.join(' ')) do
+            raw_execute(sql, except_params: except_params)
           end
         end
 
         def do_execute(sql, name = nil, format: DEFAULT_RESPONSE_FORMAT, settings: {})
-          log(sql, "#{adapter_name} #{name}") do
-            res = request(sql, format, settings)
-            process_response(res, format, sql)
-          end
+          ActiveRecord.deprecator.warn(<<~MSG.squish)
+            `do_execute` is deprecated and will be removed in an upcoming release.
+            Please use `execute` instead.
+          MSG
+          execute(sql, name, format: format, settings: settings)
         end
 
         if ::ActiveRecord::version >= Gem::Version.new('7.2')
@@ -154,7 +191,7 @@ module ActiveRecord
             if (duplicate = inserting.detect { |v| inserting.count(v) > 1 })
               raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict."
             end
-            do_execute(insert_versions_sql(inserting), nil, format: nil, settings: {max_partitions_per_insert_block: [100, inserting.size].max})
+            execute(insert_versions_sql(inserting), nil, settings: {max_partitions_per_insert_block: [100, inserting.size].max})
           end
         end
 
@@ -168,56 +205,19 @@ module ActiveRecord
           end
         end
 
-        private
-
-        # Make HTTP request to ClickHouse server
-        # @param [String] sql
-        # @param [String, nil] format
-        # @param [Hash] settings
-        # @return [Net::HTTPResponse]
-        def request(sql, format = nil, settings = {})
-          formatted_sql = apply_format(sql, format)
-          request_params = @connection_config || {}
-          @lock.synchronize do
-            @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, {
-              'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}",
-              'Content-Type' => 'application/x-www-form-urlencoded',
-            })
-          end
-        end
+        protected
 
-        def apply_format(sql, format)
-          format ? "#{sql} FORMAT #{format}" : sql
-        end
+        def table_structure(table_name)
+          result = do_system_execute("DESCRIBE TABLE `#{table_name}`", table_name)
+          data = result['data']
 
-        def process_response(res, format, sql = nil)
-          case res.code.to_i
-          when 200
-            body = res.body
+          return data unless data.empty?
 
-            if body.include?("DB::Exception") && body.match?(DB_EXCEPTION_REGEXP)
-              raise ActiveRecord::ActiveRecordError, "Response code: #{res.code}:\n#{res.body}#{sql ? "\nQuery: #{sql}" : ''}"
-            else
-              format_body_response(res.body, format)
-            end
-          else
-            case res.body
-              when /DB::Exception:.*\(UNKNOWN_DATABASE\)/
-                raise ActiveRecord::NoDatabaseError
-              when /DB::Exception:.*\(DATABASE_ALREADY_EXISTS\)/
-                raise ActiveRecord::DatabaseAlreadyExists
-              else
-                raise ActiveRecord::ActiveRecordError, "Response code: #{res.code}:\n#{res.body}"
-            end
-          end
-        rescue JSON::ParserError
-          res.body
+          raise ActiveRecord::StatementInvalid, "Could not find table '#{table_name}'"
         end
+        alias column_definitions table_structure
 
-        def log_with_debug(sql, name = nil)
-          return yield unless @debug
-          log(sql, "#{name} (system)") { yield }
-        end
+        private
 
         def schema_creation
           Clickhouse::SchemaCreation.new(self)
@@ -236,20 +236,6 @@ module ActiveRecord
           Clickhouse::Column.new(field[0], default_value, type_metadata, field[1].include?('Nullable'), default_function, codec: field[5].presence)
         end
 
-        protected
-
-        def table_structure(table_name)
-          result = do_system_execute("DESCRIBE TABLE `#{table_name}`", table_name)
-          data = result['data']
-
-          return data unless data.empty?
-
-          raise ActiveRecord::StatementInvalid, "Could not find table '#{table_name}'"
-        end
-        alias column_definitions table_structure
-
-        private
-
         # Extracts the value from a PostgreSQL column default definition.
         def extract_value_from_default(default_expression, default_type)
           return nil if default_type != 'DEFAULT' || default_expression.blank?
@@ -269,42 +255,38 @@ module ActiveRecord
           (%r{\w+\(.*\)} === default)
         end
 
-        def format_body_response(body, format)
-          return body if body.blank?
-
-          case format
-          when 'JSONCompact'
-            format_from_json_compact(body)
-          when 'JSONCompactEachRowWithNamesAndTypes'
-            format_from_json_compact_each_row_with_names_and_types(body)
-          else
-            body
-          end
-        end
-
-        def format_from_json_compact(body)
-          parse_json_payload(body)
+        def raw_execute(sql, settings: {}, except_params: [])
+          statement = Statement.new(sql, format: @response_format)
+          statement.response = request(statement, settings: settings, except_params: except_params)
+          statement.processed_response
         end
 
-        def format_from_json_compact_each_row_with_names_and_types(body)
-          rows = body.split("\n").map { |row| parse_json_payload(row) }
-          names, types, *data = rows
-
-          meta = names.zip(types).map do |name, type|
-            {
-              'name' => name,
-              'type' => type
-            }
+        # Make HTTP request to ClickHouse server
+        # @param [ActiveRecord::ConnectionAdapters::Clickhouse::Statement] statement
+        # @param [Hash] settings
+        # @param [Array] except_params
+        # @return [Net::HTTPResponse]
+        def request(statement, settings: {}, except_params: [])
+          @lock.synchronize do
+            @connection.post("/?#{settings_params(settings, except: except_params)}",
+                             statement.formatted_sql,
+                             'Content-Type' => 'application/x-www-form-urlencoded',
+                             'User-Agent' => ClickhouseAdapter::USER_AGENT)
           end
+        end
 
-          {
-            'meta' => meta,
-            'data' => data
-          }
+        def log_with_debug(sql, name = nil)
+          return yield unless @debug
+          log(sql, "#{name} (system)") { yield }
         end
 
-        def parse_json_payload(payload)
-          JSON.parse(payload, decimal_class: BigDecimal)
+        def settings_params(settings = {}, except: [])
+          request_params = @connection_config || {}
+          block_settings = @block_settings || {}
+          request_params.merge(block_settings)
+                        .merge(settings)
+                        .except(*except)
+                        .to_param
         end
       end
     end

data/lib/active_record/connection_adapters/clickhouse/statement/format_manager.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Clickhouse
+      class Statement
+        class FormatManager
+
+          def initialize(sql, format:)
+            @sql = sql.strip
+            @format = format
+          end
+
+          def apply
+            return @sql if skip_format? || @format.blank?
+
+            "#{@sql} FORMAT #{@format}"
+          end
+
+          def skip_format?
+            system_command? || schema_command? || format_specified? || delete?
+          end
+
+          private
+
+          def system_command?
+            /\Asystem|\Aoptimize/i.match?(@sql)
+          end
+
+          def schema_command?
+            /\Acreate|\Aalter|\Adrop|\Arename/i.match?(@sql)
+          end
+
+          def format_specified?
+            /format [a-z]+\z/i.match?(@sql)
+          end
+
+          def delete?
+            /\Adelete from/i.match?(@sql)
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse/statement/response_processor.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Clickhouse
+      class Statement
+        class ResponseProcessor
+
+          DB_EXCEPTION_REGEXP = /\ACode:\s+\d+\.\s+DB::Exception:/.freeze
+
+          def initialize(raw_response, format, sql)
+            @raw_response = raw_response
+            @body = raw_response.body
+            @format = format
+            @sql = sql
+          end
+
+          def process
+            if success?
+              process_successful_response
+            else
+              raise_database_error!
+            end
+          rescue JSON::ParserError
+            @body
+          end
+
+          private
+
+          def success?
+            @raw_response.code.to_i == 200
+          end
+
+          def process_successful_response
+            raise_generic!(@sql) if @body.include?('DB::Exception') && @body.match?(DB_EXCEPTION_REGEXP)
+
+            format_body_response
+          end
+
+          def raise_generic!(sql = nil)
+            raise ActiveRecord::ActiveRecordError, "Response code: #{@raw_response.code}:\n#{@body}#{"\nQuery: #{sql}" if sql}"
+          end
+
+          def format_body_response
+            return @body if @body.blank?
+
+            case @format
+            when 'JSONCompact'
+              format_from_json_compact(@body)
+            when 'JSONCompactEachRowWithNamesAndTypes'
+              format_from_json_compact_each_row_with_names_and_types(@body)
+            else
+              @body
+            end
+          rescue JSON::ParserError
+            @body
+          end
+
+          def format_from_json_compact(body)
+            parse_json_payload(body)
+          end
+
+          def format_from_json_compact_each_row_with_names_and_types(body)
+            rows = body.each_line.map { |row| parse_json_payload(row) }
+            names, types, *data = rows
+
+            meta = names.zip(types).map do |name, type|
+              {
+                'name' => name,
+                'type' => type
+              }
+            end
+
+            {
+              'meta' => meta,
+              'data' => data
+            }
+          end
+
+          def parse_json_payload(payload)
+            JSON.parse(payload, decimal_class: BigDecimal)
+          end
+
+          def raise_database_error!
+            case @body
+            when /DB::Exception:.*\(UNKNOWN_DATABASE\)/
+              raise ActiveRecord::NoDatabaseError
+            when /DB::Exception:.*\(DATABASE_ALREADY_EXISTS\)/
+              raise ActiveRecord::DatabaseAlreadyExists
+            else
+              raise_generic!
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse/statement.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'active_record/connection_adapters/clickhouse/statement/format_manager'
+require 'active_record/connection_adapters/clickhouse/statement/response_processor'
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Clickhouse
+      class Statement
+
+        attr_reader :format
+        attr_writer :response
+
+        def initialize(sql, format:)
+          @sql = sql
+          @format = format
+        end
+
+        def formatted_sql
+          @formatted_sql ||= FormatManager.new(@sql, format: @format).apply
+        end
+
+        def processed_response
+          ResponseProcessor.new(@response, @format, @sql).process
+        end
+
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -16,6 +16,7 @@ require 'active_record/connection_adapters/clickhouse/column'
 require 'active_record/connection_adapters/clickhouse/quoting'
 require 'active_record/connection_adapters/clickhouse/schema_creation'
 require 'active_record/connection_adapters/clickhouse/schema_statements'
+require 'active_record/connection_adapters/clickhouse/statement'
 require 'active_record/connection_adapters/clickhouse/table_definition'
 require 'net/http'
 require 'openssl'
@@ -82,6 +83,8 @@ module ActiveRecord
       include Clickhouse::Quoting
 
       ADAPTER_NAME = 'Clickhouse'.freeze
+      DEFAULT_RESPONSE_FORMAT = 'JSONCompactEachRowWithNamesAndTypes'.freeze
+      USER_AGENT = "ClickHouse ActiveRecord #{ClickhouseActiverecord::VERSION}"
       NATIVE_DATABASE_TYPES = {
         string: { name: 'String' },
         integer: { name: 'UInt32' },
@@ -137,6 +140,7 @@ module ActiveRecord
 
         @connection_config = { user: @config[:username], password: @config[:password], database: @config[:database] }.compact
         @debug = @config[:debug] || false
+        @response_format = @config[:format] || DEFAULT_RESPONSE_FORMAT
 
         @prepared_statements = false
 
@@ -145,7 +149,7 @@ module ActiveRecord
 
       # Return ClickHouse server version
       def server_version
-        @server_version ||= do_system_execute('SELECT version()')['data'][0][0]
+        @server_version ||= select_value('SELECT version()')
       end
 
       # Savepoints are not supported, noop
@@ -303,17 +307,14 @@ module ActiveRecord
       # @option [Boolean] single_line
       # @return [String]
       def show_create_table(table, single_line: true)
-        sql = do_system_execute("SHOW CREATE TABLE `#{table}`")['data'].try(:first).try(:first)
+        sql = do_system_execute("SHOW CREATE TABLE `#{table}`")['data'].try(:first).try(:first).gsub("#{@config[:database]}.", '')
         single_line ? sql.squish : sql
       end
 
       # Create a new ClickHouse database.
       def create_database(name)
         sql = apply_cluster "CREATE DATABASE #{quote_table_name(name)}"
-        log_with_debug(sql, adapter_name) do
-          res = @connection.post("/?#{@connection_config.except(:database).to_param}", sql)
-          process_response(res, DEFAULT_RESPONSE_FORMAT)
-        end
+        do_system_execute sql, adapter_name, except_params: [:database]
       end
 
       def create_view(table_name, request_settings: {}, **options)
@@ -326,7 +327,7 @@ module ActiveRecord
           drop_table(table_name, options.merge(if_exists: true))
         end
 
-        do_execute(schema_creation.accept(td), format: nil, settings: request_settings)
+        execute(schema_creation.accept(td), settings: request_settings)
       end
 
       def create_table(table_name, request_settings: {}, **options, &block)
@@ -343,7 +344,7 @@ module ActiveRecord
           drop_table(table_name, options.merge(if_exists: true))
         end
 
-        do_execute(schema_creation.accept(td), format: nil, settings: request_settings)
+        execute(schema_creation.accept(td), settings: request_settings)
 
         if options[:with_distributed]
           distributed_table_name = options.delete(:with_distributed)
@@ -358,16 +359,13 @@ module ActiveRecord
 
       def create_function(name, body, **options)
         fd = "CREATE#{' OR REPLACE' if options[:force]} FUNCTION #{apply_cluster(quote_table_name(name))} AS #{body}"
-        do_execute(fd, format: nil)
+        execute(fd)
       end
 
       # Drops a ClickHouse database.
       def drop_database(name) #:nodoc:
         sql = apply_cluster "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
-        log_with_debug(sql, adapter_name) do
-          res = @connection.post("/?#{@connection_config.except(:database).to_param}", sql)
-          process_response(res, DEFAULT_RESPONSE_FORMAT)
-        end
+        do_system_execute sql, adapter_name, except_params: [:database]
       end
 
       def drop_functions
@@ -377,7 +375,7 @@ module ActiveRecord
       end
 
       def rename_table(table_name, new_name)
-        do_execute apply_cluster "RENAME TABLE #{quote_table_name(table_name)} TO #{quote_table_name(new_name)}"
+        execute apply_cluster "RENAME TABLE #{quote_table_name(table_name)} TO #{quote_table_name(new_name)}"
       end
 
       def drop_table(table_name, options = {}) # :nodoc:
@@ -387,7 +385,7 @@ module ActiveRecord
         query = apply_cluster(query)
         query = "#{query} SYNC" if options[:sync]
 
-        do_execute(query)
+        execute(query)
 
         if options[:with_distributed]
           distributed_table_name = options.delete(:with_distributed)
@@ -402,25 +400,19 @@ module ActiveRecord
         query = apply_cluster(query)
         query = "#{query} SYNC" if options[:sync]
 
-        do_execute(query, format: nil)
+        execute(query)
       end
 
       def add_column(table_name, column_name, type, **options)
-        return if options[:if_not_exists] == true && column_exists?(table_name, column_name, type)
-
-        at = create_alter_table table_name
-        at.add_column(column_name, type, **options)
-        execute(schema_creation.accept(at), nil, settings: {wait_end_of_query: 1, send_progress_in_http_headers: 1})
+        with_settings(wait_end_of_query: 1, send_progress_in_http_headers: 1) { super }
       end
 
       def remove_column(table_name, column_name, type = nil, **options)
-        return if options[:if_exists] == true && !column_exists?(table_name, column_name)
-
-        execute("ALTER TABLE #{quote_table_name(table_name)} #{remove_column_for_alter(table_name, column_name, type, **options)}", nil, settings: {wait_end_of_query: 1, send_progress_in_http_headers: 1})
+        with_settings(wait_end_of_query: 1, send_progress_in_http_headers: 1) { super }
       end
 
       def change_column(table_name, column_name, type, **options)
-        result = do_execute("ALTER TABLE #{quote_table_name(table_name)} #{change_column_for_alter(table_name, column_name, type, **options)}", nil, settings: {wait_end_of_query: 1, send_progress_in_http_headers: 1})
+        result = execute("ALTER TABLE #{quote_table_name(table_name)} #{change_column_for_alter(table_name, column_name, type, **options)}", nil, settings: {wait_end_of_query: 1, send_progress_in_http_headers: 1})
         raise "Error parse json response: #{result}" if result.presence && !result.is_a?(Hash)
       end
 
@@ -479,6 +471,13 @@ module ActiveRecord
         @config[:database]
       end
 
+      # Returns the shard name from the configuration.
+      # This is used to identify the shard in replication paths when using both sharding and replication.
+      # Required when you have multiple shards with replication to ensure unique paths for each shard's replication metadata.
+      def shard
+        @config[:shard_name]
+      end
+
       def use_default_replicated_merge_tree_params?
         database_engine_atomic? && @config[:use_default_replicated_merge_tree_params]
       end
@@ -487,8 +486,17 @@ module ActiveRecord
         (replica || use_default_replicated_merge_tree_params?) && cluster
       end
 
+      # Returns the path for replication metadata.
+      # When sharding is enabled (shard_name is set), the path includes the shard identifier
+      # to ensure unique paths for each shard's replication metadata.
+      # Format with sharding: /clickhouse/tables/{cluster}/{shard}/{database}.{table}
+      # Format without sharding: /clickhouse/tables/{cluster}/{database}.{table}
       def replica_path(table)
-        "/clickhouse/tables/#{cluster}/#{@connection_config[:database]}.#{table}"
+        if shard
+          "/clickhouse/tables/#{cluster}/#{shard}/#{@connection_config[:database]}.#{table}"
+        else
+          "/clickhouse/tables/#{cluster}/#{@connection_config[:database]}.#{table}"
+        end
       end
 
       def database_engine_atomic?

data/lib/arel/visitors/clickhouse.rb

@@ -51,8 +51,14 @@ module Arel
       end
 
       def visit_Arel_Nodes_Final(o, collector)
-        visit o.expr, collector
+        visit o.expr.left, collector
         collector << ' FINAL'
+
+        o.expr.right.each do |join|
+          collector << ' '
+          visit join, collector
+        end
+
         collector
       end
 

data/lib/clickhouse-activerecord/schema_dumper.rb

@@ -31,6 +31,8 @@ module ClickhouseActiverecord
 
     def table(table, stream)
       if table.match(/^\.inner/).nil?
+        sql= ""
+        simple ||= ENV['simple'] == 'true'
         unless simple
           stream.puts "  # TABLE: #{table}"
           sql = @connection.show_create_table(table)
@@ -126,7 +128,7 @@ module ClickhouseActiverecord
       sql = @connection.show_create_function(function)
       if sql
         stream.puts "  # SQL: #{sql}"
-        stream.puts "  create_function \"#{function}\", \"#{sql.gsub(/^CREATE FUNCTION (.*?) AS/, '').strip}\", force: true"
+        stream.puts "  create_function \"#{function}\", \"#{sql.sub(/\ACREATE(OR REPLACE)? FUNCTION .*? AS/, '').strip}\", force: true"
         stream.puts
       end
     end

data/lib/clickhouse-activerecord/tasks.rb

@@ -66,9 +66,9 @@ module ClickhouseActiverecord
         if sql.gsub(/[a-z]/i, '').blank?
           next
         elsif sql =~ /^INSERT INTO/
-          connection.do_execute(sql, nil, format: nil)
+          connection.execute(sql, nil, format: nil)
         elsif sql =~ /^CREATE .*?FUNCTION/
-          connection.do_execute(sql, nil, format: nil)
+          connection.execute(sql, nil, format: nil)
         else
           connection.execute(sql)
         end

data/lib/clickhouse-activerecord/version.rb

@@ -1,3 +1,3 @@
 module ClickhouseActiverecord
-  VERSION = '1.3.0'
+  VERSION = '1.4.0'
 end

data/lib/core_extensions/active_record/relation.rb

@@ -1,6 +1,11 @@
 module CoreExtensions
   module ActiveRecord
     module Relation
+
+      def self.prepended(base)
+        base::VALID_UNSCOPING_VALUES << :final << :settings
+      end
+
       def reverse_order!
         return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
 
@@ -19,17 +24,39 @@ module CoreExtensions
       #
       # An <tt>ActiveRecord::ActiveRecordError</tt> will be raised if database not ClickHouse.
       # @param [Hash] opts
+
+
+      # Specify settings to be used for this single query.
+      # For example:
+      #
+      #   users = User.settings(use_skip_indexes: true).where(name: 'John')
+      #   # SELECT "users".* FROM "users"
+      #   # WHERE "users"."name" = 'John'
+      #   # SETTINGS use_skip_indexes = 1
       def settings(**opts)
         spawn.settings!(**opts)
       end
 
       # @param [Hash] opts
       def settings!(**opts)
-        check_command('SETTINGS')
-        @values[:settings] = (@values[:settings] || {}).merge opts
+        check_command!('SETTINGS')
+        self.settings_values = settings_values.merge opts
         self
       end
 
+      def settings_values
+        @values.fetch(:settings, ::ActiveRecord::QueryMethods::FROZEN_EMPTY_HASH)
+      end
+
+      def settings_values=(value)
+        if ::ActiveRecord::version >= Gem::Version.new('7.2')
+          assert_modifiable!
+        else
+          assert_mutability!
+        end
+        @values[:settings] = value
+      end
+
       # When FINAL is specified, ClickHouse fully merges the data before returning the result and thus performs all data transformations that happen during merges for the given table engine.
       # For example:
       #
@@ -37,16 +64,32 @@ module CoreExtensions
       #   # SELECT users.* FROM users FINAL
       #
       # An <tt>ActiveRecord::ActiveRecordError</tt> will be raised if database not ClickHouse.
-      def final
-        spawn.final!
+      #
+      # @param [Boolean] final
+      def final(final = true)
+        spawn.final!(final)
       end
 
-      def final!
-        check_command('FINAL')
-        @values[:final] = true
+      # @param [Boolean] final
+      def final!(final = true)
+        check_command!('FINAL')
+        self.final_value = final
         self
       end
 
+      def final_value=(value)
+        if ::ActiveRecord::version >= Gem::Version.new('7.2')
+          assert_modifiable!
+        else
+          assert_mutability!
+        end
+        @values[:final] = value
+      end
+
+      def final_value
+        @values.fetch(:final, nil)
+      end
+
       # GROUPING SETS allows you to specify multiple groupings in the GROUP BY clause.
       # Whereas GROUP BY CUBE generates all possible groupings, GROUP BY GROUPING SETS generates only the specified groupings.
       # For example:
@@ -128,7 +171,7 @@ module CoreExtensions
 
       private
 
-      def check_command(cmd)
+      def check_command!(cmd)
         raise ::ActiveRecord::ActiveRecordError, cmd + ' is a ClickHouse specific query clause' unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
       end
 
@@ -139,9 +182,9 @@ module CoreExtensions
           arel = super(connection_or_aliases)
         end
 
-        arel.final! if @values[:final].present?
+        arel.final! if final_value
         arel.limit_by(*@values[:limit_by]) if @values[:limit_by].present?
-        arel.settings(@values[:settings]) if @values[:settings].present?
+        arel.settings(settings_values) unless settings_values.empty?
         arel.using(@values[:using]) if @values[:using].present?
         arel.windows(@values[:windows]) if @values[:windows].present?
 

data/lib/core_extensions/arel/nodes/select_core.rb

@@ -10,6 +10,13 @@ module CoreExtensions
           ::Arel::Nodes::Final.new(super)
         end
 
+        def hash
+          [
+            @source, @set_quantifier, @projections, @optimizer_hints,
+            @wheres, @groups, @havings, @windows, @comment, @final
+          ].hash
+        end
+
         def eql?(other)
           super && final == other.final
         end

data/lib/core_extensions/arel/nodes/select_statement.rb

@@ -10,6 +10,10 @@ module CoreExtensions
           @settings = nil
         end
 
+        def hash
+          [@cores, @orders, @limit, @lock, @offset, @with, @settings].hash
+        end
+
         def eql?(other)
           super && 
             limit_by == other.limit_by &&

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clickhouse-activerecord
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Sergey Odintsov
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-01-24 00:00:00.000000000 Z
+date: 2025-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -122,6 +122,9 @@ files:
 - lib/active_record/connection_adapters/clickhouse/quoting.rb
 - lib/active_record/connection_adapters/clickhouse/schema_creation.rb
 - lib/active_record/connection_adapters/clickhouse/schema_statements.rb
+- lib/active_record/connection_adapters/clickhouse/statement.rb
+- lib/active_record/connection_adapters/clickhouse/statement/format_manager.rb
+- lib/active_record/connection_adapters/clickhouse/statement/response_processor.rb
 - lib/active_record/connection_adapters/clickhouse/table_definition.rb
 - lib/active_record/connection_adapters/clickhouse_adapter.rb
 - lib/arel/nodes/final.rb
@@ -166,7 +169,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.9
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: ClickHouse ActiveRecord