click_house 1.3.8 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c4ca9cca59d1e59aaf5e86ef624928def03e276492c775a15dbf48938716180
-  data.tar.gz: 671a7c265806735bd8ce5dec00d7dcc1b8057179a2a74b3e771ff6b7792d2864
+  metadata.gz: 402590e2de7189b46fca70711b33bf8428ccbc677d8d98ad4205b470bcf5eeb4
+  data.tar.gz: b3a33602650115c46c638347b993d118921e7254a2ed9f20bdb92ad30c50a05a
 SHA512:
-  metadata.gz: 8c191e44fd27d8b08bbb2be2cca11057421557526596ca05cc8086182bbbb8a31897ccc92c3475221fdcd7a3c8fc6caeab312f4ffbbb7a313a2014d461542c16
-  data.tar.gz: cfed5341b0320d362b05023ce48de8400a5078337f52d9adde990eb53465e9e1a15f388ae72ebf3a7335e4a3f1762cbc6c3a0197d3fb00c4f2e42def6a1157f0
+  metadata.gz: 4d71565e61efb57711463fa3b4071d418ad8136656d64e3de15fbb7f4b48134ab284e9b2dc13b0dcbdf77dd9bee19aa4757593b0dee98480ac6cd64cc4e85a5f
+  data.tar.gz: c2d45bd01a211acfb8be077d38ac93807c191c95a3a6fe2a6ac63a7a097b5c599351f0c6282b86d533eff2dd59d2820c0f824ffd975d0cd6692636eb0bb31e48

data/.github/workflows/main.yml

@@ -8,13 +8,13 @@ jobs:
 
     services:
       clickhouse:
-        image: yandex/clickhouse-server:20.12.3.3
+        image: yandex/clickhouse-server:22.1
         ports: 
           - 8123:8123
 
     strategy:
       matrix:
-        ruby-version: [2.7, 2.6, 2.5, 2.4]
+        ruby-version: [3.1, 3.0, 2.7, 2.6]
 
     steps:
       - uses: actions/checkout@v2

data/.gitignore

@@ -1,5 +1,6 @@
 /.bundle/
 /.yardoc
+/.idea/
 /_yardoc/
 /coverage/
 /pkg/

data/.rubocop.yml

@@ -13,9 +13,9 @@ AllCops:
 Bundler/OrderedGems:
   Enabled: false
 
-# ============================== Documentation ======================
-Style/Documentation:
-  Enabled: false
+# ============================== Gemspec ======================
+Gemspec/DateAssignment:
+  Enabled: true
 
 # =============================== Performance =======================
 Performance/AncestorsInclude:
@@ -46,6 +46,12 @@ Performance/ConstantRegexp:
   Enabled: true
 Performance/MethodObjectAsBlock:
   Enabled: false
+Performance/RedundantEqualityComparisonBlock:
+  Enabled: true
+Performance/RedundantSplitRegexpArgument:
+  Enabled: true
+Performance/MapCompact:
+  Enabled: true
 
 # ============================== Metrics ============================
 Metrics/ClassLength:
@@ -69,6 +75,8 @@ Naming/MethodParameterName:
   Enabled: false
 Naming/AccessorMethodName:
   Enabled: false
+Naming/InclusiveLanguage:
+  Enabled: true
 
 # ============================== Layout =============================
 Layout/LineLength:
@@ -105,6 +113,10 @@ Layout/EmptyLinesAroundAttributeAccessor:
   Enabled: true
 Layout/SpaceAroundMethodCallOperator:
   Enabled: true
+Layout/SpaceBeforeBrackets:
+  Enabled: true
+Layout/LineEndStringConcatenationIndentation:
+  Enabled: true
 
 # ============================== Style ==============================
 Style/RescueModifier:
@@ -217,6 +229,24 @@ Style/SwapValues:
   Enabled: true
 Style/RedundantArgument:
   Enabled: true
+Style/HashExcept:
+  Enabled: true
+Style/EndlessMethod:
+  Enabled: true
+Style/IfWithBooleanLiteralBranches:
+  Enabled: true
+Style/HashConversion:
+  Enabled: true
+Style/Documentation:
+  Enabled: false
+Style/InPatternThen:
+  Enabled: true
+Style/MultilineInPatternThen:
+  Enabled: true
+Style/QuotedSymbols:
+  Enabled: true
+Style/StringChars:
+  Enabled: true
 
 # ============================== Lint ==============================
 Lint/DuplicateMethods:
@@ -287,3 +317,21 @@ Lint/UnmodifiedReduceAccumulator:
   Enabled: true
 Lint/UnexpectedBlockArity:
   Enabled: true
+Lint/DeprecatedConstants:
+  Enabled: true
+Lint/LambdaWithoutLiteralBlock:
+  Enabled: true
+Lint/NumberedParameterAssignment:
+  Enabled: true
+Lint/OrAssignmentToConstant:
+  Enabled: true
+Lint/RedundantDirGlobSort:
+  Enabled: true
+Lint/SymbolConversion:
+  Enabled: true
+Lint/TripleQuotes:
+  Enabled: true
+Lint/AmbiguousAssignment:
+  Enabled: true
+Lint/EmptyInPattern:
+  Enabled: true

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+# 1.6.0
+* [PR](https://github.com/shlima/click_house/pull/19) handle value returned as nil in float and integer types (case of Aggregate Function Combinators) 
+* [PR](https://github.com/shlima/click_house/pull/18) Fix Faraday deprecation
+
+# 1.5.0
+* add support for 'WITH TOTALS' modifier in response
+* send SQL in GET request's body [#12](https://github.com/shlima/click_house/pull/12)
+* add support of 'WITH TOTALS' on a resulting set
+
+# 1.4.0
+* fix decimal type casting [#11](https://github.com/shlima/click_house/issues/11)
+
+# 1.3.9
+* add `ClickHouse.connection.add_index`, `ClickHouse.connection.drop_index`
+
 # 1.3.8
 * fix `DateTime` casting for queries like `ClickHouse.connection.select_value('select NOW()')` 
 * fix resulting set console inspection
@@ -9,7 +24,7 @@
 * fix ruby 2.7 warning `maybe ** should be added to the call` on `ClickHouse.connection.databases`
 
 # 1.3.5
-* added `ClickHouse.connexction.explain("sql")` 
+* added `ClickHouse.connection.explain("sql")` 
 
 # 1.3.4
 * added `ClickHouse.type_names(nullable: false)`

data/Gemfile.lock

@@ -1,63 +1,83 @@
 PATH
   remote: .
   specs:
-    click_house (1.3.8)
-      faraday
+    click_house (1.5.0)
+      faraday (>= 1.7)
       faraday_middleware
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ast (2.4.1)
+    ast (2.4.2)
     coderay (1.1.3)
-    diff-lcs (1.4.4)
-    faraday (1.1.0)
+    diff-lcs (1.5.0)
+    faraday (1.9.3)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0)
+      faraday-multipart (~> 1.0)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.0)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
+      faraday-retry (~> 1.0)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
+    faraday-multipart (1.0.3)
       multipart-post (>= 1.2, < 3)
-      ruby2_keywords
-    faraday_middleware (1.0.0)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
+    faraday-retry (1.0.3)
+    faraday_middleware (1.2.0)
       faraday (~> 1.0)
     method_source (1.0.0)
     multipart-post (2.1.1)
-    parallel (1.20.1)
-    parser (2.7.2.0)
+    parallel (1.21.0)
+    parser (3.1.0.0)
       ast (~> 2.4.1)
-    pry (0.13.1)
+    pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    rainbow (3.0.0)
-    rake (13.0.3)
-    regexp_parser (2.0.1)
-    rexml (3.2.4)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    regexp_parser (2.2.0)
+    rexml (3.2.5)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
       rspec-expectations (~> 3.10.0)
       rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.0)
+    rspec-core (3.10.1)
       rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.0)
+    rspec-expectations (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.0)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-support (3.10.0)
-    rubocop (1.6.1)
+    rspec-support (3.10.3)
+    rubocop (1.25.0)
       parallel (~> 1.10)
-      parser (>= 2.7.1.5)
+      parser (>= 3.1.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.2.0, < 2.0)
+      rubocop-ast (>= 1.15.1, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (1.3.0)
-      parser (>= 2.7.1.5)
-    rubocop-performance (1.9.1)
-      rubocop (>= 0.90.0, < 2.0)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.15.1)
+      parser (>= 3.0.1.1)
+    rubocop-performance (1.13.2)
+      rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
-    ruby-progressbar (1.10.1)
-    ruby2_keywords (0.0.2)
-    unicode-display_width (1.7.0)
+    ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
+    unicode-display_width (2.1.0)
 
 PLATFORMS
   ruby
@@ -72,4 +92,4 @@ DEPENDENCIES
   rubocop-performance
 
 BUNDLED WITH
-   2.1.4
+   2.3.3

data/README.md

@@ -104,6 +104,9 @@ ClickHouse.connection.truncate_tables(['table_1', 'table_2'], if_exists: true, c
 ClickHouse.connection.truncate_tables # will truncate all tables in database
 ClickHouse.connection.rename_table('old_name', 'new_name', cluster: nil)
 ClickHouse.connection.rename_table(%w[table_1 table_2], %w[new_1 new_2], cluster: nil)
+ClickHouse.connection.alter_table('table', 'DROP COLUMN user_id', cluster: nil)
+ClickHouse.connection.add_index('table', 'ix', 'has(b, a)', type: 'minmax', granularity: 2, cluster: nil)
+ClickHouse.connection.drop_index('table', 'ix', cluster: nil)
 
 ClickHouse.connection.select_all('SELECT * FROM visits')
 ClickHouse.connection.select_one('SELECT * FROM visits LIMIT 1')
@@ -497,6 +500,10 @@ class ClickHouseRecord < ActiveRecord::Base
     def select_all
       agent.select_all(current_scope.to_sql)
     end
+
+    def explain
+      agent.explain(current_scope.to_sql)
+    end
   end
 end
 ````

data/click_house.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.summary       = 'Modern Ruby database driver for ClickHouse'
   spec.description   = 'Yandex ClickHouse database interface for Ruby'
   spec.homepage      = 'https://github.com/shlima/click_house'
-  spec.required_ruby_version = '>= 2.4.0'
+  spec.required_ruby_version = '>= 2.6.0'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'faraday'
+  spec.add_dependency 'faraday', '>= 1.7'
   spec.add_dependency 'faraday_middleware'
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'

data/docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.5'
 
 services:
   clickhouse:
-    image: yandex/clickhouse-server:20.12.3.3
+    image: yandex/clickhouse-server:22.1
     ports:
       - "8123:8123"
       - "9000:9000"

data/lib/click_house/config.rb

@@ -49,11 +49,15 @@ module ClickHouse
     end
 
     def logger!
-      @logger || Logger.new(IO::NULL)
+      @logger || null_logger
     end
 
     def url!
       @url || "#{scheme}://#{host}:#{port}"
     end
+
+    def null_logger
+      @null_logger ||= Logger.new(IO::NULL)
+    end
   end
 end

data/lib/click_house/connection.rb

@@ -21,8 +21,25 @@ module ClickHouse
       post(body, query: { query: query }, database: database)
     end
 
-    def get(path = '/', query: {}, database: config.database)
-      transport.get(compose(path, query.merge(database: database)))
+    # @param path [String] Clickhouse HTTP endpoint, e.g. /ping, /replica_status
+    # @param body [String] SQL to run
+    # @param database [String|NilClass] database to use, nil to skip
+    # @param query [Hash] other CH settings to send through params, e.g. max_rows_to_read=1
+    # @example get(body: 'select number from system.numbers limit 100', query: { max_rows_to_read: 10 })
+    # @return [Faraday::Response]
+    def get(path = '/', body: '', query: {}, database: config.database)
+      # backward compatibility since
+      # https://github.com/shlima/click_house/pull/12/files#diff-9c6f3f06d3b575731eae4b6b95ddbcdcc20452c432b8f6e87a3a8e8645818107R24
+      if query.is_a?(String)
+        query = { query: query }
+        config.logger!.warn('since v1.4.0 use connection.get(body: "SELECT 1") instead of connection.get(query: "SELECT 1")')
+      end
+
+      transport.get(path) do |conn|
+        conn.params = query.merge(database: database).compact
+        conn.params[:send_progress_in_http_headers] = 1 unless body.empty?
+        conn.body = body
+      end
     end
 
     def post(body = nil, query: {}, database: config.database)
@@ -35,7 +52,7 @@ module ClickHouse
         conn.options.open_timeout = config.open_timeout
         conn.headers = config.headers
         conn.ssl.verify = config.ssl_verify
-        conn.basic_auth(config.username, config.password) if config.auth?
+        conn.request(:basic_auth, config.username, config.password) if config.auth?
         conn.response Middleware::Logging, logger: config.logger!
         conn.response Middleware::RaiseError
         conn.response :json, content_type: %r{application/json}

data/lib/click_house/extend/connection_altering.rb

@@ -66,6 +66,33 @@ module ClickHouse
 
         execute(format(template, pattern)).success?
       end
+
+      def add_index(
+        table_name,
+        name,
+        expression,
+        type:,
+        granularity: nil,
+        after: nil,
+        cluster: nil
+      )
+        template = 'ADD INDEX %<name>s %<expression>s TYPE %<type>s GRANULARITY %<granularity>d %<after>s'
+        pattern = {
+          name: name,
+          expression: expression,
+          type: type,
+          granularity: granularity,
+          after: Util::Statement.ensure(after, "AFTER #{after}"),
+        }
+
+        alter_table(table_name, format(template, pattern), cluster: cluster)
+      end
+
+      def drop_index(table_name, name, cluster: nil)
+        alter_table(table_name, <<~SQL, cluster: cluster)
+          DROP INDEX #{name}
+        SQL
+      end
     end
   end
 end

data/lib/click_house/extend/connection_explaining.rb

@@ -6,9 +6,11 @@ module ClickHouse
       EXPLAIN = 'EXPLAIN'
       EXPLAIN_RE = /\A(\s*#{EXPLAIN})/io.freeze
 
-      def explain(sql, io: $stdout)
+      # @return String
+      def explain(sql, io: StringIO.new)
         res = execute("#{EXPLAIN} #{sql.gsub(EXPLAIN_RE, '')}")
-        io << res.body
+        io.puts(res.body)
+        io.string
       end
     end
   end

data/lib/click_house/extend/connection_healthy.rb

@@ -4,8 +4,7 @@ module ClickHouse
   module Extend
     module ConnectionHealthy
       def ping
-        # without +send_progress_in_http_headers: nil+ DB::Exception: Empty query returns
-        get(database: nil, query: { send_progress_in_http_headers: nil }).success?
+        get('/ping', database: nil).success?
       end
 
       def replicas_status

data/lib/click_house/extend/connection_selective.rb

@@ -5,17 +5,17 @@ module ClickHouse
     module ConnectionSelective
       # @return [ResultSet]
       def select_all(sql)
-        response = execute(Util::Statement.format(sql, 'JSON'))
+        response = get(body: Util::Statement.format(sql, 'JSON'))
         Response::Factory[response]
       end
 
       def select_value(sql)
-        response = execute(Util::Statement.format(sql, 'JSON'))
+        response = get(body: Util::Statement.format(sql, 'JSON'))
         Array(Response::Factory[response].first).dig(0, -1)
       end
 
       def select_one(sql)
-        response = execute(Util::Statement.format(sql, 'JSON'))
+        response = get(body: Util::Statement.format(sql, 'JSON'))
         Response::Factory[response].first
       end
     end

data/lib/click_house/middleware/logging.rb

@@ -29,11 +29,8 @@ module ClickHouse
       # rubocop:disable Layout/LineLength
       def on_complete(env)
         summary = extract_summary(env.response_headers)
-        elapsed = duration
-        query = CGI.parse(env.url.query.to_s).dig('query', 0) || '[NO QUERY]'
-
-        logger.info("\e[1mSQL (#{Util::Pretty.measure(elapsed)})\e[0m #{query};")
-        logger.debug(body) if body
+        logger.info("\e[1mSQL (#{duration_stats_log(env.body)})\e[0m #{query(env)};")
+        logger.debug(body) if body && !query_in_body?(env)
         logger.info("\e[1mRead: #{summary.fetch(:read_rows)} rows, #{summary.fetch(:read_bytes)}. Written: #{summary.fetch(:written_rows)} rows, #{summary.fetch(:written_bytes)}\e[0m")
       end
       # rubocop:enable Layout/LineLength
@@ -46,6 +43,28 @@ module ClickHouse
         Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end
 
+      def query_in_body?(env)
+        env.method == :get
+      end
+
+      def query(env)
+        if query_in_body?(env)
+          body
+        else
+          CGI.parse(env.url.query.to_s).dig('query', 0) || '[NO QUERY]'
+        end
+      end
+
+      def duration_stats_log(body)
+        elapsed = duration
+        clickhouse_elapsed = body['statistics'].fetch('elapsed') if body.is_a?(Hash) && body.key?('statistics')
+
+        [
+          "Total: #{Util::Pretty.measure(elapsed * 1000)}",
+          ("CH: #{Util::Pretty.measure(clickhouse_elapsed * 1000)}" if clickhouse_elapsed)
+        ].compact.join(', ')
+      end
+
       def extract_summary(headers)
         JSON.parse(headers.fetch('x-clickhouse-summary', '{}')).tap do |summary|
           summary[:read_rows] = summary['read_rows']

data/lib/click_house/response/factory.rb

@@ -10,7 +10,12 @@ module ClickHouse
 
         return body if !body.is_a?(Hash) || !(body.key?('meta') && body.key?('data'))
 
-        ResultSet.new(meta: body.fetch('meta'), data: body.fetch('data'), statistics: body['statistics'])
+        ResultSet.new(
+          meta: body.fetch('meta'),
+          data: body.fetch('data'),
+          totals: body['totals'],
+          statistics: body['statistics']
+        )
       end
     end
   end

data/lib/click_house/response/result_set.rb

@@ -14,7 +14,7 @@ module ClickHouse
                      :inspect, :each, :fetch, :length, :count, :size,
                      :first, :last, :[], :to_h
 
-      attr_reader :meta, :data, :statistics
+      attr_reader :meta, :data, :totals, :statistics
 
       class << self
         # @return [Array<String, Array>]
@@ -48,9 +48,13 @@ module ClickHouse
 
       # @param meta [Array]
       # @param data [Array]
-      def initialize(meta:, data:, statistics: nil)
+      # @param totals [Array|Hash|NilClass] Support for 'GROUP BY WITH TOTALS' modifier
+      #   https://clickhouse.tech/docs/en/sql-reference/statements/select/group-by/#with-totals-modifier
+      #   Hash in JSON format and Array in JSONCompact
+      def initialize(meta:, data:, totals: nil, statistics: nil)
         @meta = meta
         @data = data
+        @totals = totals
         @statistics = Hash(statistics)
       end
 

data/lib/click_house/type/decimal_type.rb

@@ -4,11 +4,11 @@ module ClickHouse
   module Type
     class DecimalType < BaseType
       def cast(value, precision = Float::DIG, _scale = nil)
-        BigDecimal(value, precision.to_f)
+        BigDecimal(value, precision.to_i)
       end
 
       def serialize(value, precision = Float::DIG, _scale = nil)
-        BigDecimal(value, precision.to_f) unless value.nil?
+        BigDecimal(value, precision.to_i).to_f unless value.nil?
       end
     end
   end

data/lib/click_house/type/float_type.rb

@@ -4,7 +4,7 @@ module ClickHouse
   module Type
     class FloatType < BaseType
       def cast(value)
-        Float(value)
+        Float(value) unless value.nil?
       end
 
       def serialize(value)

data/lib/click_house/type/integer_type.rb

@@ -4,7 +4,7 @@ module ClickHouse
   module Type
     class IntegerType < BaseType
       def cast(value)
-        Integer(value)
+        Integer(value) unless value.nil?
       end
 
       def serialize(value)

data/lib/click_house/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ClickHouse
-  VERSION = '1.3.8'
+  VERSION = '1.6.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: click_house
 version: !ruby/object:Gem::Version
-  version: 1.3.8
+  version: 1.6.0
 platform: ruby
 authors:
 - Aliaksandr Shylau
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-12-21 00:00:00.000000000 Z
+date: 2022-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: faraday_middleware
   requirement: !ruby/object:Gem::Requirement
@@ -203,14 +203,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.0
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.3
 signing_key:
 specification_version: 4
 summary: Modern Ruby database driver for ClickHouse