clickhouse-activerecord 1.1.2 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7c1652d6cf040ebaf43559441ccfa2496f4c408b232ca6d3d1c71da9bbc20209
-  data.tar.gz: 900d15344b100536b6f1e0ac1dd3e1df03b890fe5c1b6f7b6f9218283156c17f
+  metadata.gz: c3301f161ab9a3c94507a1d3a989e32710628746eeeac895966e64dd25579dd4
+  data.tar.gz: 2ab20aabc31a593987fb8905b5c58f023025a24cf447d0d922644ebbcff63397
 SHA512:
-  metadata.gz: d9a74ee0ca1e865f821d5c8a1f8612e30dcd790cd79725344ab568b246d30d895711a69f5f05542ff1989f194605a70d7906ec432c90bf2c0688f5a5b120b3ab
-  data.tar.gz: ddb82724f85c5a2cfb6da4db250d20f9fc6eb7cddf91a8bada889f07eb717f8657ad011d71e2ab19cb625b83b74dad91a8adeb1bcf94886f33fbd2cd999cc833
+  metadata.gz: f2d35a1489031909c0a2b8c2f8feb5624b93854bfbcabd9e39e006874033d0b534d568e72deabb795fcbeb1c0a3416966df8168b6f10d1ef64a3bb36d96b838f
+  data.tar.gz: 9c6be79ff71e1207cd2b162e909db8b7db8a2276ce17351b2b2fc38097dc47f8c4dd8386cfacdc37bd1ccd3cfc28072dc45bd09b155737ef28b63a3a55f35def

data/.github/workflows/testing.yml

@@ -27,7 +27,7 @@ jobs:
           - ruby: 3.2
             rails: 7.1.3
           - ruby: 3.2
-            rails: 7.2.0
+            rails: 7.2.1
         clickhouse: [ '22.1', '24.6' ]
 
     steps:
@@ -49,7 +49,7 @@ jobs:
           ruby-version: ${{ matrix.version.ruby }}
           bundler-cache: true
 
-      - run: bundle exec rspec spec/single
+      - run: bundle exec rspec spec/single --format progress
 
   tests_cluster:
     name: Testing cluster server
@@ -94,4 +94,4 @@ jobs:
           ruby-version: ${{ matrix.version.ruby }}
           bundler-cache: true
 
-      - run: bundle exec rspec spec/cluster
+      - run: bundle exec rspec spec/cluster --format progress

data/README.md

@@ -237,12 +237,13 @@ class CreateDataItems < ActiveRecord::Migration[7.1]
 end
 ```
 
-Create table with custom column structure:
+Create table with custom column structure and codec compression:
 
 ```ruby
 class CreateDataItems < ActiveRecord::Migration[7.1]
   def change
     create_table "data_items", id: false, options: "MergeTree PARTITION BY toYYYYMM(timestamp) ORDER BY timestamp", force: :cascade do |t|
+      t.integer :user_id, limit: 8, codec: 'DoubleDelta, LZ4'
       t.column "timestamp", "DateTime('UTC') CODEC(DoubleDelta, LZ4)"
     end
   end

data/lib/active_record/connection_adapters/clickhouse/column.rb

@@ -0,0 +1,21 @@
+module ActiveRecord
+  module ConnectionAdapters
+    module Clickhouse
+      class Column < ActiveRecord::ConnectionAdapters::Column
+
+        attr_reader :codec
+
+        def initialize(name, default, sql_type_metadata = nil, null = true, default_function = nil, codec: nil, **args)
+          super
+          @codec = codec
+        end
+
+        private
+
+        def deduplicated
+          self
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse/oid/map.rb

@@ -26,6 +26,8 @@ module ActiveRecord
           def deserialize(value)
             if value.is_a?(::Hash)
               value.map { |k, item| [k.to_s, deserialize(item)] }.to_h
+            elsif value.is_a?(::Array)
+              value.map { |item| deserialize(item) }
             else
               return value if value.nil?
               case @subtype
@@ -44,6 +46,8 @@ module ActiveRecord
           def serialize(value)
             if value.is_a?(::Hash)
               value.map { |k, item| [k.to_s, serialize(item)] }.to_h
+            elsif value.is_a?(::Array)
+              value.map { |item| serialize(item) }
             else
               return value if value.nil?
               case @subtype

data/lib/active_record/connection_adapters/clickhouse/schema_creation.rb

@@ -33,9 +33,15 @@ module ActiveRecord
           if options[:array]
             sql.gsub!(/\s+(.*)/, ' Array(\1)')
           end
-          if options[:map]
+          if options[:map] == :array
+            sql.gsub!(/\s+(.*)/, ' Map(String, Array(\1))')
+          end
+          if options[:map] == true
             sql.gsub!(/\s+(.*)/, ' Map(String, \1)')
           end
+          if options[:codec]
+            sql.gsub!(/\s+(.*)/, " \\1 CODEC(#{options[:codec]})")
+          end
           sql.gsub!(/(\sString)\(\d+\)/, '\1')
           sql << " DEFAULT #{quote_default_expression(options[:default], options[:column])}" if options_include_default?(options)
           sql

data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb

@@ -8,6 +8,8 @@ module ActiveRecord
       module SchemaStatements
         DEFAULT_RESPONSE_FORMAT = 'JSONCompactEachRowWithNamesAndTypes'.freeze
 
+        DB_EXCEPTION_REGEXP = /\ACode:\s+\d+\.\s+DB::Exception:/.freeze
+
         def execute(sql, name = nil, settings: {})
           do_execute(sql, name, settings: settings)
         end
@@ -183,7 +185,9 @@ module ActiveRecord
         def process_response(res, format, sql = nil)
           case res.code.to_i
           when 200
-            if res.body.to_s.include?("DB::Exception")
+            body = res.body
+
+            if body.include?("DB::Exception") && body.match?(DB_EXCEPTION_REGEXP)
               raise ActiveRecord::ActiveRecordError, "Response code: #{res.code}:\n#{res.body}#{sql ? "\nQuery: #{sql}" : ''}"
             else
               format_body_response(res.body, format)
@@ -221,7 +225,7 @@ module ActiveRecord
           default_value = extract_value_from_default(field[3], field[2])
           default_function = extract_default_function(field[3])
           default_value = lookup_cast_type(sql_type).cast(default_value)
-          ClickhouseColumn.new(field[0], default_value, type_metadata, field[1].include?('Nullable'), default_function)
+          Clickhouse::Column.new(field[0], default_value, type_metadata, field[1].include?('Nullable'), default_function, codec: field[5].presence)
         end
 
         protected

data/lib/active_record/connection_adapters/clickhouse/{schema_definitions.rb → table_definition.rb}

@@ -94,10 +94,15 @@ module ActiveRecord
           args.each { |name| column(name, kind, **options.except(:limit)) }
         end
 
+        def column(name, type, index: nil, **options)
+          options[:null] = false if type.match?(/Nullable\([^)]+\)/)
+          super(name, type, index: index, **options)
+        end
+
         private
 
         def valid_column_definition_options
-          super + [:array, :low_cardinality, :fixed_string, :value, :type, :map]
+          super + [:array, :low_cardinality, :fixed_string, :value, :type, :map, :codec]
         end
       end
 

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -2,18 +2,21 @@
 
 require 'arel/visitors/clickhouse'
 require 'arel/nodes/final'
+require 'arel/nodes/grouping_sets'
 require 'arel/nodes/settings'
 require 'arel/nodes/using'
+require 'arel/nodes/limit_by'
 require 'active_record/connection_adapters/clickhouse/oid/array'
 require 'active_record/connection_adapters/clickhouse/oid/date'
 require 'active_record/connection_adapters/clickhouse/oid/date_time'
 require 'active_record/connection_adapters/clickhouse/oid/big_integer'
 require 'active_record/connection_adapters/clickhouse/oid/map'
 require 'active_record/connection_adapters/clickhouse/oid/uuid'
+require 'active_record/connection_adapters/clickhouse/column'
 require 'active_record/connection_adapters/clickhouse/quoting'
-require 'active_record/connection_adapters/clickhouse/schema_definitions'
 require 'active_record/connection_adapters/clickhouse/schema_creation'
 require 'active_record/connection_adapters/clickhouse/schema_statements'
+require 'active_record/connection_adapters/clickhouse/table_definition'
 require 'net/http'
 require 'openssl'
 
@@ -47,7 +50,12 @@ module ActiveRecord
 
   module ModelSchema
     module ClassMethods
-      delegate :final, :final!, :settings, :settings!, :window, :window!, to: :all
+      delegate :final, :final!,
+               :group_by_grouping_sets, :group_by_grouping_sets!,
+               :settings, :settings!,
+               :window, :window!,
+               :limit_by, :limit_by!,
+               to: :all
 
       def is_view
         @is_view || false
@@ -70,13 +78,6 @@ module ActiveRecord
       register "clickhouse", "ActiveRecord::ConnectionAdapters::ClickhouseAdapter", "active_record/connection_adapters/clickhouse_adapter"
     end
 
-    class ClickhouseColumn < Column
-      private
-      def deduplicated
-        self
-      end
-    end
-
     class ClickhouseAdapter < AbstractAdapter
       include Clickhouse::Quoting
 
@@ -351,8 +352,8 @@ module ActiveRecord
         end
       end
 
-      def create_function(name, body)
-        fd = "CREATE FUNCTION #{apply_cluster(quote_table_name(name))} AS #{body}"
+      def create_function(name, body, **options)
+        fd = "CREATE#{' OR REPLACE' if options[:force]} FUNCTION #{apply_cluster(quote_table_name(name))} AS #{body}"
         do_execute(fd, format: nil)
       end
 

data/lib/arel/nodes/grouping_sets.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Arel # :nodoc: all
+  module Nodes
+    class GroupingSets < Arel::Nodes::Unary
+
+      def initialize(expr)
+        super
+        @expr = wrap_grouping_sets(expr)
+      end
+
+      private
+
+      def wrap_grouping_sets(sets)
+        sets.map do |element|
+          # See Arel::SelectManager#group
+          case element
+          when Array
+            wrap_grouping_sets(element)
+          when String
+            ::Arel::Nodes::SqlLiteral.new(element)
+          when Symbol
+            ::Arel::Nodes::SqlLiteral.new(element.to_s)
+          else
+            element
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/arel/nodes/limit_by.rb

@@ -0,0 +1,17 @@
+module Arel # :nodoc: all
+  module Nodes
+    class LimitBy < Arel::Nodes::Unary
+      attr_reader :column
+
+      def initialize(limit, column)
+        raise ArgumentError, 'Limit should be an integer' unless limit.is_a?(Integer)
+        raise ArgumentError, 'Limit should be a positive integer' unless limit >= 0
+        raise ArgumentError, 'Column should be a Symbol or String' unless column.is_a?(String) || column.is_a?(Symbol)
+        
+        @column = column
+
+        super(limit)
+      end
+    end
+  end
+end

data/lib/arel/visitors/clickhouse.rb

@@ -25,6 +25,7 @@ module Arel
       end
 
       def visit_Arel_Nodes_SelectOptions(o, collector)
+        maybe_visit o.limit_by, collector
         maybe_visit o.settings, super
       end
 
@@ -45,6 +46,11 @@ module Arel
         collector
       end
 
+      def visit_Arel_Nodes_GroupingSets(o, collector)
+        collector << 'GROUPING SETS '
+        grouping_array_or_grouping_element(o.expr, collector)
+      end
+
       def visit_Arel_Nodes_Settings(o, collector)
         return collector if o.expr.empty?
 
@@ -64,6 +70,11 @@ module Arel
         collector
       end
 
+      def visit_Arel_Nodes_LimitBy(o, collector)
+        collector << "LIMIT #{o.expr} BY #{o.column}"
+        collector
+      end
+
       def visit_Arel_Nodes_Matches(o, collector)
         op = o.case_sensitive ? " LIKE " : " ILIKE "
         infix_value o, collector, op
@@ -95,6 +106,25 @@ module Arel
         @connection.sanitize_as_setting_name(value)
       end
 
+      private
+
+      # Utilized by GroupingSet, Cube & RollUp visitors to
+      # handle grouping aggregation semantics
+      def grouping_array_or_grouping_element(o, collector)
+        if o.is_a? Array
+          collector << '( '
+          o.each_with_index do |el, i|
+            collector << ', ' if i > 0
+            grouping_array_or_grouping_element el, collector
+          end
+          collector << ' )'
+        elsif o.respond_to? :expr
+          visit o.expr, collector
+        else
+          visit o, collector
+        end
+      end
+
     end
   end
 end

data/lib/clickhouse-activerecord/schema_dumper.rb

@@ -35,7 +35,7 @@ module ClickhouseActiverecord
           # super(table.gsub(/^\.inner\./, ''), stream)
 
           # detect view table
-          match = sql.match(/^CREATE\s+(MATERIALIZED\s+)?VIEW/)
+          view_match = sql.match(/^CREATE\s+(MATERIALIZED\s+)?VIEW\s+\S+\s+(?:TO (\S+))?/)
         end
 
         # Copy from original dumper
@@ -50,20 +50,16 @@ module ClickhouseActiverecord
 
           unless simple
             # Add materialize flag
-            tbl.print ', view: true' if match
-            tbl.print ', materialized: true' if match && match[1].presence
+            tbl.print ', view: true' if view_match
+            tbl.print ', materialized: true' if view_match && view_match[1].presence
+            tbl.print ", to: \"#{view_match[2]}\"" if view_match && view_match[2].presence
           end
 
-          case pk
-          when String
-            tbl.print ", primary_key: #{pk.inspect}" unless pk == "id"
-            pkcol = columns.detect { |c| c.name == pk }
-            pkcolspec = column_spec_for_primary_key(pkcol)
-            if pkcolspec.present?
-              tbl.print ", #{format_colspec(pkcolspec)}"
+          if (id = columns.detect { |c| c.name == 'id' })
+            spec = column_spec_for_primary_key(id)
+            if spec.present?
+              tbl.print ", #{format_colspec(spec)}"
             end
-          when Array
-            tbl.print ", primary_key: #{pk.inspect}"
           else
             tbl.print ", id: false"
           end
@@ -80,10 +76,10 @@ module ClickhouseActiverecord
           tbl.puts ", force: :cascade do |t|"
 
           # then dump all non-primary key columns
-          if simple || !match
+          if simple || !view_match
             columns.each do |column|
               raise StandardError, "Unknown type '#{column.sql_type}' for column '#{column.name}'" unless @connection.valid_type?(column.type)
-              next if column.name == pk
+              next if column.name == pk && column.name == "id"
               type, colspec = column_spec(column)
               name = column.name =~ (/\./) ? "\"`#{column.name}`\"" : column.name.inspect
               tbl.print "    t.#{type} #{name}"
@@ -116,8 +112,11 @@ module ClickhouseActiverecord
     def function(function, stream)
       stream.puts "  # FUNCTION: #{function}"
       sql = @connection.show_create_function(function)
-      stream.puts "  # SQL: #{sql}" if sql
-      stream.puts "  create_function \"#{function}\", \"#{sql.gsub(/^CREATE FUNCTION (.*?) AS/, '').strip}\"" if sql
+      if sql
+        stream.puts "  # SQL: #{sql}"
+        stream.puts "  create_function \"#{function}\", \"#{sql.gsub(/^CREATE FUNCTION (.*?) AS/, '').strip}\", force: true"
+        stream.puts
+      end
     end
 
     def format_options(options)
@@ -146,23 +145,32 @@ module ClickhouseActiverecord
     end
 
     def schema_array(column)
-      (column.sql_type =~ /Array?\(/).nil? ? nil : true
+      (column.sql_type =~ /Array\(/).nil? ? nil : true
     end
 
     def schema_map(column)
-      (column.sql_type =~ /Map?\(/).nil? ? nil : true
+      if column.sql_type =~ /Map\(([^,]+),\s*(Array)\)/
+        return :array
+      end
+
+      (column.sql_type =~ /Map\(/).nil? ? nil : true
     end
 
     def schema_low_cardinality(column)
-      (column.sql_type =~ /LowCardinality?\(/).nil? ? nil : true
+      (column.sql_type =~ /LowCardinality\(/).nil? ? nil : true
     end
 
+    # @param [ActiveRecord::ConnectionAdapters::Clickhouse::Column] column
     def prepare_column_options(column)
       spec = {}
       spec[:unsigned] = schema_unsigned(column)
       spec[:array] = schema_array(column)
       spec[:map] = schema_map(column)
+      if spec[:map] == :array
+        spec[:array] = nil
+      end
       spec[:low_cardinality] = schema_low_cardinality(column)
+      spec[:codec] = column.codec.inspect if column.codec
       spec.merge(super).compact
     end
 

data/lib/clickhouse-activerecord/tasks.rb

@@ -67,6 +67,8 @@ module ClickhouseActiverecord
           next
         elsif sql =~ /^INSERT INTO/
           connection.do_execute(sql, nil, format: nil)
+        elsif sql =~ /^CREATE .*?FUNCTION/
+          connection.do_execute(sql, nil, format: nil)
         else
           connection.execute(sql)
         end

data/lib/clickhouse-activerecord/version.rb

@@ -1,3 +1,3 @@
 module ClickhouseActiverecord
-  VERSION = '1.1.2'
+  VERSION = '1.2.0'
 end

data/lib/core_extensions/active_record/relation.rb

@@ -47,6 +47,33 @@ module CoreExtensions
         self
       end
 
+      # GROUPING SETS allows you to specify multiple groupings in the GROUP BY clause.
+      # Whereas GROUP BY CUBE generates all possible groupings, GROUP BY GROUPING SETS generates only the specified groupings.
+      # For example:
+      #
+      #   users = User.group_by_grouping_sets([], [:name], [:name, :age]).select(:name, :age, 'count(*)')
+      #   # SELECT name, age, count(*) FROM users GROUP BY GROUPING SETS ( (), (name), (name, age) )
+      #
+      # which is generally equivalent to:
+      #   # SELECT NULL, NULL, count(*) FROM users
+      #   # UNION ALL
+      #   # SELECT name, NULL, count(*) FROM users GROUP BY name
+      #   # UNION ALL
+      #   # SELECT name, age, count(*) FROM users GROUP BY name, age
+      #
+      # Raises <tt>ArgumentError</tt> if no grouping sets are specified are provided.
+      def group_by_grouping_sets(*grouping_sets)
+        raise ArgumentError, 'The method .group_by_grouping_sets() must contain arguments.' if grouping_sets.blank?
+
+        spawn.group_by_grouping_sets!(*grouping_sets)
+      end
+
+      def group_by_grouping_sets!(*grouping_sets) # :nodoc:
+        grouping_sets = grouping_sets.map { |set| arel_columns(set) }
+        self.group_values += [::Arel::Nodes::GroupingSets.new(grouping_sets)]
+        self
+      end
+
       # The USING clause specifies one or more columns to join, which establishes the equality of these columns. For example:
       #
       #   users = User.joins(:joins).using(:event_name, :date)
@@ -81,6 +108,24 @@ module CoreExtensions
         self
       end
 
+      # The LIMIT BY clause permit to improve deduplication based on a unique key, it has better performances than
+      # the GROUP BY clause
+      #
+      #   users = User.limit_by(1, id)
+      #   # SELECT users.* FROM users LIMIT 1 BY id
+      #
+      # An <tt>ActiveRecord::ActiveRecordError</tt> will be reaised if database is not Clickhouse.
+      # @param [Array] opts
+      def limit_by(*opts)
+        spawn.limit_by!(*opts)
+      end
+
+      # @param [Array] opts
+      def limit_by!(*opts)
+        @values[:limit_by] = *opts
+        self
+      end
+
       private
 
       def check_command(cmd)
@@ -95,6 +140,7 @@ module CoreExtensions
         end
 
         arel.final! if @values[:final].present?
+        arel.limit_by(*@values[:limit_by]) if @values[:limit_by].present?
         arel.settings(@values[:settings]) if @values[:settings].present?
         arel.using(@values[:using]) if @values[:using].present?
         arel.windows(@values[:windows]) if @values[:windows].present?

data/lib/core_extensions/arel/nodes/select_statement.rb

@@ -2,15 +2,18 @@ module CoreExtensions
   module Arel # :nodoc: all
     module Nodes
       module SelectStatement
-        attr_accessor :settings
+        attr_accessor :limit_by, :settings
 
         def initialize(relation = nil)
           super
+          @limit_by = nil
           @settings = nil
         end
 
         def eql?(other)
-          super && settings == other.settings
+          super && 
+            limit_by == other.limit_by &&
+            settings == other.settings
         end
       end
     end

data/lib/core_extensions/arel/select_manager.rb

@@ -29,6 +29,11 @@ module CoreExtensions
         @ctx.source.right.last.right = ::Arel::Nodes::Using.new(::Arel.sql(exprs.join(',')))
         self
       end
+
+      def limit_by(*exprs)
+        @ast.limit_by = ::Arel::Nodes::LimitBy.new(*exprs)
+        self
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clickhouse-activerecord
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.2.0
 platform: ruby
 authors:
 - Sergey Odintsov
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-08-27 00:00:00.000000000 Z
+date: 2024-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -106,6 +106,7 @@ files:
 - bin/console
 - bin/setup
 - clickhouse-activerecord.gemspec
+- lib/active_record/connection_adapters/clickhouse/column.rb
 - lib/active_record/connection_adapters/clickhouse/oid/array.rb
 - lib/active_record/connection_adapters/clickhouse/oid/big_integer.rb
 - lib/active_record/connection_adapters/clickhouse/oid/date.rb
@@ -114,10 +115,12 @@ files:
 - lib/active_record/connection_adapters/clickhouse/oid/uuid.rb
 - lib/active_record/connection_adapters/clickhouse/quoting.rb
 - lib/active_record/connection_adapters/clickhouse/schema_creation.rb
-- lib/active_record/connection_adapters/clickhouse/schema_definitions.rb
 - lib/active_record/connection_adapters/clickhouse/schema_statements.rb
+- lib/active_record/connection_adapters/clickhouse/table_definition.rb
 - lib/active_record/connection_adapters/clickhouse_adapter.rb
 - lib/arel/nodes/final.rb
+- lib/arel/nodes/grouping_sets.rb
+- lib/arel/nodes/limit_by.rb
 - lib/arel/nodes/settings.rb
 - lib/arel/nodes/using.rb
 - lib/arel/visitors/clickhouse.rb