clickhouse-activerecord 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3301f161ab9a3c94507a1d3a989e32710628746eeeac895966e64dd25579dd4
-  data.tar.gz: 2ab20aabc31a593987fb8905b5c58f023025a24cf447d0d922644ebbcff63397
+  metadata.gz: 8e47b553f60d06d69a8e1f7c70bbc495c32502f0ea75f3ede9c8ac2bafb063ee
+  data.tar.gz: 42ec02c29478ae577fda1f1a726d003e925b2c36faa02fd6a92797a4e05f960c
 SHA512:
-  metadata.gz: f2d35a1489031909c0a2b8c2f8feb5624b93854bfbcabd9e39e006874033d0b534d568e72deabb795fcbeb1c0a3416966df8168b6f10d1ef64a3bb36d96b838f
-  data.tar.gz: 9c6be79ff71e1207cd2b162e909db8b7db8a2276ce17351b2b2fc38097dc47f8c4dd8386cfacdc37bd1ccd3cfc28072dc45bd09b155737ef28b63a3a55f35def
+  metadata.gz: 405f20e2c53352ae702b4b462165ccbf5306da2e72733d5c8f80588ba5392637e6b8d2c95d6d61cc9f777d0fc49e48a3578ee6db2fdb292668cb4ec0c52d0471
+  data.tar.gz: 8b3bebc0c3672112c26889555d1eb0b280cbc331758cbf1d5d1733273983e10178929bec76fda6793a654d1ff2c0048b44a58eca533f3ec0a2e8481e7d928314

data/.github/workflows/testing.yml

@@ -28,13 +28,15 @@ jobs:
             rails: 7.1.3
           - ruby: 3.2
             rails: 7.2.1
-        clickhouse: [ '22.1', '24.6' ]
+          - ruby: 3.2
+            rails: 8.0.1
+        clickhouse: [ '22.1', '24.9' ]
 
     steps:
       - uses: actions/checkout@v4
 
       - name: Start ClickHouse ${{ matrix.clickhouse }}
-        uses: isbang/compose-action@v1.5.1
+        uses: hoverkraft-tech/compose-action@v2.1.0
         env:
           CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
         with:
@@ -72,14 +74,16 @@ jobs:
           - ruby: 3.2
             rails: 7.1.3
           - ruby: 3.2
-            rails: 7.2.0
-        clickhouse: [ '22.1', '24.6' ]
+            rails: 7.2.1
+          - ruby: 3.2
+            rails: 8.0.1
+        clickhouse: [ '22.1', '24.9' ]
 
     steps:
       - uses: actions/checkout@v4
 
       - name: Start ClickHouse Cluster ${{ matrix.clickhouse }}
-        uses: isbang/compose-action@v1.5.1
+        uses: hoverkraft-tech/compose-action@v2.1.0
         env:
           CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
         with:

data/README.md

@@ -184,7 +184,11 @@ Action.settings(optimize_read_in_order: 1).where(date: Date.current).limit(10)
 
 User.joins(:actions).using(:group_id)
 # Clickhouse User Load (10.3ms)  SELECT users.* FROM users INNER JOIN actions USING group_id
-#=> #<ActiveRecord::Relation [#<Action *** >]>
+#=> #<ActiveRecord::Relation [#<User *** >]>
+
+User.window('x', order: 'date', partition: 'name', rows: 'UNBOUNDED PRECEDING').select('sum(value) OVER x')
+# SELECT sum(value) OVER x FROM users WINDOW x AS (PARTITION BY name ORDER BY date ROWS UNBOUNDED PRECEDING)
+#=> #<ActiveRecord::Relation [#<User *** >]>
 ```
 
 

data/clickhouse-activerecord.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_runtime_dependency 'bundler', '>= 1.13.4'
-  spec.add_runtime_dependency 'activerecord', '~> 7.1'
+  spec.add_runtime_dependency 'activerecord', '>= 7.1', '< 9.0'
 
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.4'

data/lib/active_record/connection_adapters/clickhouse/oid/map.rb

@@ -7,15 +7,16 @@ module ActiveRecord
         class Map < Type::Value # :nodoc:
 
           def initialize(sql_type)
-            @subtype = case sql_type
-                       when /U?Int\d+/
-                         :integer
-                       when /DateTime/
-                         :datetime
-                       when /Date/
-                         :date
-                       else
-                         :string
+            case sql_type
+            when /U?Int(\d+)/
+              @subtype = :integer
+              @limit = bits_to_limit(Regexp.last_match(1)&.to_i)
+            when /DateTime/
+              @subtype = :datetime
+            when /Date/
+              @subtype = :date
+            else
+              @subtype = :string
             end
           end
 
@@ -65,6 +66,19 @@ module ActiveRecord
             end
           end
 
+          private
+
+          def bits_to_limit(bits)
+            case bits
+            when 8   then 1
+            when 16  then 2
+            when 32  then 4
+            when 64  then 8
+            when 128 then 16
+            when 256 then 32
+            end
+          end
+
         end
       end
     end

data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb

@@ -72,8 +72,14 @@ module ActiveRecord
           result['data'].flatten
         end
 
+        def materialized_views(name = nil)
+          result = do_system_execute("SHOW TABLES WHERE engine = 'MaterializedView'", name)
+          return [] if result.nil?
+          result['data'].flatten
+        end
+
         def functions
-          result = do_system_execute("SELECT name FROM system.functions WHERE origin = 'SQLUserDefined'")
+          result = do_system_execute("SELECT name FROM system.functions WHERE origin = 'SQLUserDefined' ORDER BY name")
           return [] if result.nil?
           result['data'].flatten
         end
@@ -172,10 +178,12 @@ module ActiveRecord
         def request(sql, format = nil, settings = {})
           formatted_sql = apply_format(sql, format)
           request_params = @connection_config || {}
-          @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, {
-            'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}",
-            'Content-Type' => 'application/x-www-form-urlencoded',
-          })
+          @lock.synchronize do
+            @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, {
+              'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}",
+              'Content-Type' => 'application/x-www-form-urlencoded',
+            })
+          end
         end
 
         def apply_format(sql, format)

data/lib/active_record/connection_adapters/clickhouse/table_definition.rb

@@ -102,7 +102,7 @@ module ActiveRecord
         private
 
         def valid_column_definition_options
-          super + [:array, :low_cardinality, :fixed_string, :value, :type, :map, :codec]
+          super + [:array, :low_cardinality, :fixed_string, :value, :type, :map, :codec, :unsigned]
         end
       end
 

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -191,6 +191,8 @@ module ActiveRecord
               nil
             when /(Nullable)?\(?U?Int64\)?/
               8
+            when /(Nullable)?\(?U?Int128\)?/
+              16
             else
               super
           end
@@ -298,9 +300,11 @@ module ActiveRecord
       end
 
       # @param [String] table
+      # @option [Boolean] single_line
       # @return [String]
-      def show_create_table(table)
-        do_system_execute("SHOW CREATE TABLE `#{table}`")['data'].try(:first).try(:first).gsub(/[\n\s]+/m, ' ').gsub("#{@config[:database]}.", "")
+      def show_create_table(table, single_line: true)
+        sql = do_system_execute("SHOW CREATE TABLE `#{table}`")['data'].try(:first).try(:first)
+        single_line ? sql.squish : sql
       end
 
       # Create a new ClickHouse database.
@@ -312,7 +316,7 @@ module ActiveRecord
         end
       end
 
-      def create_view(table_name, **options)
+      def create_view(table_name, request_settings: {}, **options)
         options.merge!(view: true)
         options = apply_replica(table_name, options)
         td = create_table_definition(apply_cluster(table_name), **options)
@@ -322,10 +326,10 @@ module ActiveRecord
           drop_table(table_name, options.merge(if_exists: true))
         end
 
-        do_execute(schema_creation.accept(td), format: nil)
+        do_execute(schema_creation.accept(td), format: nil, settings: request_settings)
       end
 
-      def create_table(table_name, **options, &block)
+      def create_table(table_name, request_settings: {}, **options, &block)
         options = apply_replica(table_name, options)
         td = create_table_definition(apply_cluster(table_name), **options)
         block.call td if block_given?
@@ -339,7 +343,7 @@ module ActiveRecord
           drop_table(table_name, options.merge(if_exists: true))
         end
 
-        do_execute(schema_creation.accept(td), format: nil)
+        do_execute(schema_creation.accept(td), format: nil, settings: request_settings)
 
         if options[:with_distributed]
           distributed_table_name = options.delete(:with_distributed)

data/lib/arel/visitors/clickhouse.rb

@@ -4,6 +4,11 @@ module Arel
   module Visitors
     class Clickhouse < ::Arel::Visitors::ToSql
 
+      def compile(node, collector = Arel::Collectors::SQLString.new)
+        @delete_or_update = false
+        super
+      end
+
       def aggregate(name, o, collector)
         # replacing function name for materialized view
         if o.expressions.first && o.expressions.first != '*' && !o.expressions.first.is_a?(String) && o.expressions.first.relation&.is_view
@@ -16,12 +21,11 @@ module Arel
       # https://clickhouse.com/docs/en/sql-reference/statements/delete
       # DELETE and UPDATE in ClickHouse working only without table name
       def visit_Arel_Attributes_Attribute(o, collector)
-        if collector.value.is_a?(String)
-          collector << quote_table_name(o.relation.table_alias || o.relation.name) << '.' unless collector.value.start_with?('DELETE FROM ') || collector.value.include?(' UPDATE ')
-          collector << quote_column_name(o.name)
-        else
-          super
+        unless @delete_or_update
+          join_name  = o.relation.table_alias || o.relation.name
+          collector << quote_table_name(join_name) << '.'
         end
+        collector << quote_column_name(o.name)
       end
 
       def visit_Arel_Nodes_SelectOptions(o, collector)
@@ -30,6 +34,7 @@ module Arel
       end
 
       def visit_Arel_Nodes_UpdateStatement(o, collector)
+        @delete_or_update = true
         o = prepare_update_statement(o)
 
         collector << 'ALTER TABLE '
@@ -40,6 +45,11 @@ module Arel
         maybe_visit o.limit, collector
       end
 
+      def visit_Arel_Nodes_DeleteStatement(o, collector)
+        @delete_or_update = true
+        super
+      end
+
       def visit_Arel_Nodes_Final(o, collector)
         visit o.expr, collector
         collector << ' FINAL'
@@ -64,7 +74,7 @@ module Arel
         collector
       end
 
-      def visit_Arel_Nodes_Using o, collector
+      def visit_Arel_Nodes_Using(o, collector)
         collector << "USING "
         visit o.expr, collector
         collector

data/lib/clickhouse-activerecord/schema_dumper.rb

@@ -15,13 +15,16 @@ module ClickhouseActiverecord
     private
 
     def tables(stream)
-      functions = @connection.functions
+      functions = @connection.functions.sort
       functions.each do |function|
         function(function, stream)
       end
 
-      sorted_tables = @connection.tables.sort {|a,b| @connection.show_create_table(a).match(/^CREATE\s+(MATERIALIZED\s+)?VIEW/) ? 1 : a <=> b }
-      sorted_tables.each do |table_name|
+      view_tables = @connection.views.sort
+      materialized_view_tables = @connection.materialized_views.sort
+      sorted_tables = @connection.tables.sort - view_tables - materialized_view_tables
+
+      (sorted_tables + view_tables + materialized_view_tables).each do |table_name|
         table(table_name, stream) unless ignored?(table_name)
       end
     end
@@ -109,6 +112,15 @@ module ClickhouseActiverecord
       end
     end
 
+    def column_spec_for_primary_key(column)
+      spec = super
+
+      id = ActiveRecord::ConnectionAdapters::ClickhouseAdapter::NATIVE_DATABASE_TYPES.invert[{name: column.sql_type.gsub(/\(\d+\)/, "")}]
+      spec[:id] = id.inspect if id.present?
+
+      spec.except!(:limit, :unsigned) # This can be removed at some date, it is only here to clean up existing schemas which have dumped these values already
+    end
+
     def function(function, stream)
       stream.puts "  # FUNCTION: #{function}"
       sql = @connection.show_create_function(function)

data/lib/clickhouse-activerecord/tasks.rb

@@ -40,19 +40,19 @@ module ClickhouseActiverecord
       # get all tables
       tables = connection.execute("SHOW TABLES FROM #{@configuration.database} WHERE name NOT LIKE '.inner_id.%'")['data'].flatten.map do |table|
         next if %w[schema_migrations ar_internal_metadata].include?(table)
-        connection.show_create_table(table).gsub("#{@configuration.database}.", '')
+        connection.show_create_table(table, single_line: false).gsub("#{@configuration.database}.", '')
       end.compact
 
       # sort view to last
       tables.sort_by! {|table| table.match(/^CREATE\s+(MATERIALIZED\s+)?VIEW/) ? 1 : 0}
 
       # get all functions
-      functions = connection.execute("SELECT create_query FROM system.functions WHERE origin = 'SQLUserDefined'")['data'].flatten
+      functions = connection.execute("SELECT create_query FROM system.functions WHERE origin = 'SQLUserDefined' ORDER BY name")['data'].flatten
 
       # put to file
       File.open(args.first, 'w:utf-8') do |file|
         functions.each do |function|
-          file.puts function + ";\n\n"
+          file.puts function.gsub('\\n', "\n") + ";\n\n"
         end
 
         tables.each do |table|

data/lib/clickhouse-activerecord/version.rb

@@ -1,3 +1,3 @@
 module ClickhouseActiverecord
-  VERSION = '1.2.0'
+  VERSION = '1.3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clickhouse-activerecord
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Sergey Odintsov
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-23 00:00:00.000000000 Z
+date: 2025-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -28,16 +28,22 @@ dependencies:
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '7.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '7.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement