click_house 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '095426f805a20574d725842799b3e09368ef4239bf8a262d4ca061ef7689cedd'
-  data.tar.gz: 77098569ba1c05b236bc6b68118116c63985c1cc91bc49cdc5ae49c2550459f6
+  metadata.gz: 3dbe7ae01f5130dc0d18d9f9456ad75c2fdcc04bfe815debac344c93ce9bd00e
+  data.tar.gz: 4f7c8ab45593d9d37744ea9125c202052a99f582cd0887e251af790973c8a22a
 SHA512:
-  metadata.gz: a71819d5ea5c61bf85907501e9b80ca713dc3e78e97a066a9f3e8fb1713dfc0a81b91aef0b12c51389afad53c518ccca913544be0f44d60182d02ecb722e0e1c
-  data.tar.gz: 95fb2a123492cb6f816ed4c8b1984ec1ba21b1784eceb489543081616ad5d0a5947d56623be63c7e8d5f4586977f4c2999a0bb8c229d20519e8d1f8fc2365749
+  metadata.gz: f865c9753769cfa587ce59e428d45a664d1b976b535418e86bba7688ab3b0255177d9de7201479a683635c7376ae0958ad7803490328f55f003ba10ec098c2fc
+  data.tar.gz: ae3cf74cc163b842f9e27d49c84165b94858fe9b91461aa6a0bfde7404d9c9175a663a099c75e9eaf4e05ec8a1bc2dbf1840e0ed1f26f48590dc217dd6a8b4e1

data/.github/workflows/main.yml

@@ -8,7 +8,7 @@ jobs:
 
     services:
       clickhouse:
-        image: yandex/clickhouse-server:21.1.4
+        image: yandex/clickhouse-server:21.3
         ports: 
           - 8123:8123
 

data/.gitignore

@@ -1,5 +1,6 @@
 /.bundle/
 /.yardoc
+/.idea/
 /_yardoc/
 /coverage/
 /pkg/

data/.rubocop.yml

@@ -13,9 +13,9 @@ AllCops:
 Bundler/OrderedGems:
   Enabled: false
 
-# ============================== Documentation ======================
-Style/Documentation:
-  Enabled: false
+# ============================== Gemspec ======================
+Gemspec/DateAssignment:
+  Enabled: true
 
 # =============================== Performance =======================
 Performance/AncestorsInclude:
@@ -46,6 +46,10 @@ Performance/ConstantRegexp:
   Enabled: true
 Performance/MethodObjectAsBlock:
   Enabled: false
+Performance/RedundantEqualityComparisonBlock:
+  Enabled: true
+Performance/RedundantSplitRegexpArgument:
+  Enabled: true
 
 # ============================== Metrics ============================
 Metrics/ClassLength:
@@ -227,6 +231,10 @@ Style/EndlessMethod:
   Enabled: true
 Style/IfWithBooleanLiteralBranches:
   Enabled: true
+Style/HashConversion:
+  Enabled: true
+Style/Documentation:
+  Enabled: false
 
 # ============================== Lint ==============================
 Lint/DuplicateMethods:

data/CHANGELOG.md

@@ -1,5 +1,10 @@
+# 1.5.0
+* add support for 'WITH TOTALS' modifier in response
+* send SQL in GET request's body [#12](https://github.com/shlima/click_house/pull/12)
+* add support of 'WITH TOTALS' on a resulting set
+
 # 1.4.0
-* fizx decimal type casting [#11](https://github.com/shlima/click_house/issues/11)
+* fix decimal type casting [#11](https://github.com/shlima/click_house/issues/11)
 
 # 1.3.9
 * add `ClickHouse.connection.add_index`, `ClickHouse.connection.drop_index`

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    click_house (1.4.0)
+    click_house (1.5.0)
       faraday
       faraday_middleware
 
@@ -11,11 +11,15 @@ GEM
     ast (2.4.2)
     coderay (1.1.3)
     diff-lcs (1.4.4)
-    faraday (1.3.0)
+    faraday (1.4.1)
+      faraday-excon (~> 1.1)
       faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
       multipart-post (>= 1.2, < 3)
-      ruby2_keywords
+      ruby2_keywords (>= 0.0.4)
+    faraday-excon (1.1.0)
     faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.1.0)
     faraday_middleware (1.0.0)
       faraday (~> 1.0)
     method_source (1.0.0)
@@ -28,7 +32,7 @@ GEM
       method_source (~> 1.0)
     rainbow (3.0.0)
     rake (13.0.3)
-    regexp_parser (2.0.3)
+    regexp_parser (2.1.1)
     rexml (3.2.4)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
@@ -43,7 +47,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
     rspec-support (3.10.2)
-    rubocop (1.9.1)
+    rubocop (1.11.0)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
@@ -54,7 +58,7 @@ GEM
       unicode-display_width (>= 1.4.0, < 3.0)
     rubocop-ast (1.4.1)
       parser (>= 2.7.1.5)
-    rubocop-performance (1.9.2)
+    rubocop-performance (1.10.1)
       rubocop (>= 0.90.0, < 2.0)
       rubocop-ast (>= 0.4.0)
     ruby-progressbar (1.11.0)
@@ -74,4 +78,4 @@ DEPENDENCIES
   rubocop-performance
 
 BUNDLED WITH
-   2.2.3
+   2.2.14

data/docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.5'
 
 services:
   clickhouse:
-    image: yandex/clickhouse-server:21.1.4
+    image: yandex/clickhouse-server:21.3
     ports:
       - "8123:8123"
       - "9000:9000"

data/lib/click_house/config.rb

@@ -49,11 +49,15 @@ module ClickHouse
     end
 
     def logger!
-      @logger || Logger.new(IO::NULL)
+      @logger || null_logger
     end
 
     def url!
       @url || "#{scheme}://#{host}:#{port}"
     end
+
+    def null_logger
+      @null_logger ||= Logger.new(IO::NULL)
+    end
   end
 end

data/lib/click_house/connection.rb

@@ -21,8 +21,25 @@ module ClickHouse
       post(body, query: { query: query }, database: database)
     end
 
-    def get(path = '/', query: {}, database: config.database)
-      transport.get(compose(path, query.merge(database: database)))
+    # @param path [String] Clickhouse HTTP endpoint, e.g. /ping, /replica_status
+    # @param body [String] SQL to run
+    # @param database [String|NilClass] database to use, nil to skip
+    # @param query [Hash] other CH settings to send through params, e.g. max_rows_to_read=1
+    # @example get(body: 'select number from system.numbers limit 100', query: { max_rows_to_read: 10 })
+    # @return [Faraday::Response]
+    def get(path = '/', body: '', query: {}, database: config.database)
+      # backward compatibility since
+      # https://github.com/shlima/click_house/pull/12/files#diff-9c6f3f06d3b575731eae4b6b95ddbcdcc20452c432b8f6e87a3a8e8645818107R24
+      if query.is_a?(String)
+        query = { query: query }
+        config.logger!.warn('since v1.4.0 use connection.get(body: "SELECT 1") instead of connection.get(query: "SELECT 1")')
+      end
+
+      transport.get(path) do |conn|
+        conn.params = query.merge(database: database).compact
+        conn.params[:send_progress_in_http_headers] = 1 unless body.empty?
+        conn.body = body
+      end
     end
 
     def post(body = nil, query: {}, database: config.database)

data/lib/click_house/extend/connection_explaining.rb

@@ -6,9 +6,11 @@ module ClickHouse
       EXPLAIN = 'EXPLAIN'
       EXPLAIN_RE = /\A(\s*#{EXPLAIN})/io.freeze
 
-      def explain(sql, io: $stdout)
+      # @return String
+      def explain(sql, io: StringIO.new)
         res = execute("#{EXPLAIN} #{sql.gsub(EXPLAIN_RE, '')}")
-        io << res.body
+        io.puts(res.body)
+        io.string
       end
     end
   end

data/lib/click_house/extend/connection_healthy.rb

@@ -4,8 +4,7 @@ module ClickHouse
   module Extend
     module ConnectionHealthy
       def ping
-        # without +send_progress_in_http_headers: nil+ DB::Exception: Empty query returns
-        get(database: nil, query: { send_progress_in_http_headers: nil }).success?
+        get('/ping', database: nil).success?
       end
 
       def replicas_status

data/lib/click_house/extend/connection_selective.rb

@@ -5,17 +5,17 @@ module ClickHouse
     module ConnectionSelective
       # @return [ResultSet]
       def select_all(sql)
-        response = execute(Util::Statement.format(sql, 'JSON'))
+        response = get(body: Util::Statement.format(sql, 'JSON'))
         Response::Factory[response]
       end
 
       def select_value(sql)
-        response = execute(Util::Statement.format(sql, 'JSON'))
+        response = get(body: Util::Statement.format(sql, 'JSON'))
         Array(Response::Factory[response].first).dig(0, -1)
       end
 
       def select_one(sql)
-        response = execute(Util::Statement.format(sql, 'JSON'))
+        response = get(body: Util::Statement.format(sql, 'JSON'))
         Response::Factory[response].first
       end
     end

data/lib/click_house/middleware/logging.rb

@@ -29,11 +29,8 @@ module ClickHouse
       # rubocop:disable Layout/LineLength
       def on_complete(env)
         summary = extract_summary(env.response_headers)
-        elapsed = duration
-        query = CGI.parse(env.url.query.to_s).dig('query', 0) || '[NO QUERY]'
-
-        logger.info("\e[1mSQL (#{Util::Pretty.measure(elapsed)})\e[0m #{query};")
-        logger.debug(body) if body
+        logger.info("\e[1mSQL (#{duration_stats_log(env.body)})\e[0m #{query(env)};")
+        logger.debug(body) if body && !query_in_body?(env)
         logger.info("\e[1mRead: #{summary.fetch(:read_rows)} rows, #{summary.fetch(:read_bytes)}. Written: #{summary.fetch(:written_rows)} rows, #{summary.fetch(:written_bytes)}\e[0m")
       end
       # rubocop:enable Layout/LineLength
@@ -46,6 +43,28 @@ module ClickHouse
         Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end
 
+      def query_in_body?(env)
+        env.method == :get
+      end
+
+      def query(env)
+        if query_in_body?(env)
+          body
+        else
+          CGI.parse(env.url.query.to_s).dig('query', 0) || '[NO QUERY]'
+        end
+      end
+
+      def duration_stats_log(body)
+        elapsed = duration
+        clickhouse_elapsed = body['statistics'].fetch('elapsed') if body.is_a?(Hash) && body.key?('statistics')
+
+        [
+          "Total: #{Util::Pretty.measure(elapsed * 1000)}",
+          ("CH: #{Util::Pretty.measure(clickhouse_elapsed * 1000)}" if clickhouse_elapsed)
+        ].compact.join(', ')
+      end
+
       def extract_summary(headers)
         JSON.parse(headers.fetch('x-clickhouse-summary', '{}')).tap do |summary|
           summary[:read_rows] = summary['read_rows']

data/lib/click_house/response/factory.rb

@@ -10,7 +10,12 @@ module ClickHouse
 
         return body if !body.is_a?(Hash) || !(body.key?('meta') && body.key?('data'))
 
-        ResultSet.new(meta: body.fetch('meta'), data: body.fetch('data'), statistics: body['statistics'])
+        ResultSet.new(
+          meta: body.fetch('meta'),
+          data: body.fetch('data'),
+          totals: body['totals'],
+          statistics: body['statistics']
+        )
       end
     end
   end

data/lib/click_house/response/result_set.rb

@@ -14,7 +14,7 @@ module ClickHouse
                      :inspect, :each, :fetch, :length, :count, :size,
                      :first, :last, :[], :to_h
 
-      attr_reader :meta, :data, :statistics
+      attr_reader :meta, :data, :totals, :statistics
 
       class << self
         # @return [Array<String, Array>]
@@ -48,9 +48,13 @@ module ClickHouse
 
       # @param meta [Array]
       # @param data [Array]
-      def initialize(meta:, data:, statistics: nil)
+      # @param totals [Array|Hash|NilClass] Support for 'GROUP BY WITH TOTALS' modifier
+      #   https://clickhouse.tech/docs/en/sql-reference/statements/select/group-by/#with-totals-modifier
+      #   Hash in JSON format and Array in JSONCompact
+      def initialize(meta:, data:, totals: nil, statistics: nil)
         @meta = meta
         @data = data
+        @totals = totals
         @statistics = Hash(statistics)
       end
 

data/lib/click_house/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ClickHouse
-  VERSION = '1.4.0'
+  VERSION = '1.5.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: click_house
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Aliaksandr Shylau
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-02-15 00:00:00.000000000 Z
+date: 2021-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
@@ -210,7 +210,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.2.15
 signing_key:
 specification_version: 4
 summary: Modern Ruby database driver for ClickHouse