click_house 2.1.0 → 2.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f271ed936f2d0f66ec2045ec72025e317523dc8d70123c2bcf145ede5b4f1871
-  data.tar.gz: d3e48437b08cbf5eb96315bd46ba88429302bcbebc882d787257c3636c586c8f
+  metadata.gz: 364117bb31fd3478fcf00d981b61205aa9fb864abbfeb9f9bdf253e53a0917fa
+  data.tar.gz: d0b0d7a21175a9c2e044a88351ddd6d107b1875769ae737d3e547e41e01d93d1
 SHA512:
-  metadata.gz: 3bae70c6d33a93ad298db0d14b91beb4bbfe31117cb51e9a58cdeeb95dc48e506363d3e660e6b9e3cd54398df2065210f3b0582e1891bb76e2bdd900520269b3
-  data.tar.gz: cb202cf091808068383a32bb3a2a078aaacc3e051e5c30ecfc47d0bdfab84bb188f0ace4dbc6dbd2950407e5743deeb5f2ffd82c3b31a27ef6fcea01a3fdab1d
+  metadata.gz: 481ba8166f2300302c1a0836e336097e97a25015fbe5a5cf8e8fb5912b2170d50a2a1bc25178deb8b4a4b17d63e5638d0200bbca5a84c6a237de694ecf5f4d76
+  data.tar.gz: ba051b743929517452507d54f98de6d57de724b4400d1cf9db02666a48c5b00d4dbd00a3ffab6b5f2f9c3a7db78c15f84ae36523e2ee76946526fe131722ee05

data/CHANGELOG.md

@@ -1,5 +1,9 @@
+# 2.1.1
+* Fix logging with symbolized keys JSON
+* Unknown formats return raw `Response::ResultSelt` like regular JSON query
+
 # 2.1.0
-* `ClickHouse.connection.insert` now returns `ClickHouse::Response::Execution` objects
+* `ClickHouse.connection.insert` now returns `ClickHouse::Response::Summary` object
   with methods `headers`, `summary`, `written_rows`, `written_bytes`, etc... 
 * `ClickHouse.connection.insert(columns: ["id"], values: [1])` now uses `JSONCompactEachRow` by default
   (to increase JSON serialization speed)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    click_house (2.1.0)
+    click_house (2.1.1)
       activesupport
       faraday (>= 1.7, < 3)
 

data/Gemfile_faraday1.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    click_house (2.1.0)
+    click_house (2.1.1)
       activesupport
       faraday (>= 1.7, < 3)
 

data/Gemfile_faraday2.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    click_house (2.1.0)
+    click_house (2.1.1)
       activesupport
       faraday (>= 1.7, < 3)
 

data/README.md

@@ -138,12 +138,16 @@ Select all type-casted result set
 @result = ClickHouse.connection.select_all('SELECT * FROM visits')
 
 # all enumerable methods are delegated like #each, #map, #select etc
+# results of #to_a is type casted
 @result.to_a #=> [{"date"=>#<Date: 2000-01-01>, "id"=>1}]
 
 # you can access raw data
 @result.meta #=> [{"name"=>"date", "type"=>"Date"}, {"name"=>"id", "type"=>"UInt32"}]
 @result.data #=> [{"date"=>"2000-01-01", "id"=>1}, {"date"=>"2000-01-02", "id"=>2}]
 @result.statistics #=> {"elapsed"=>0.0002271, "rows_read"=>2, "bytes_read"=>12}
+@result.summary #=> ClickHouse::Response::Summary
+@result.headers #=> {"x-clickhouse-query-id"=>"9bf5f604-31fc-4eff-a4b5-277f2c71d199"}
+@result.types #=> [Hash<String|Symbol, ClickHouse::Ast::Statement>]
 ```
 
 ### Select Value

data/lib/click_house/connection.rb

@@ -65,8 +65,9 @@ module ClickHouse
 
         conn.response Middleware::RaiseError
         conn.response Middleware::Logging, logger: config.logger!
-        conn.response config.json_parser, content_type: %r{application/json}, parser_options: { config: config }
-        conn.response Middleware::ParseCsv, content_type: %r{text/csv}, parser_options: { config: config }
+        conn.response Middleware::SummaryMiddleware, options: { config: config } # should be after logger
+        conn.response config.json_parser, content_type: %r{application/json}, options: { config: config }
+        conn.response Middleware::ParseCsv, content_type: %r{text/csv}, options: { config: config }
         conn.adapter config.adapter
       end
     end

data/lib/click_house/extend/connection_inserting.rb

@@ -46,7 +46,7 @@ module ClickHouse
           return insert_compact(table, columns: columns, values: values, format: format)
         end
 
-        Response::Factory.empty_exec
+        Response::Factory.empty_exec(config)
       end
       # rubocop:enable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
 

data/lib/click_house/extend/connection_selective.rb

@@ -11,7 +11,16 @@ module ClickHouse
 
       def select_value(sql)
         response = get(body: sql, query: { default_format: 'JSON' })
-        Array(Response::Factory.response(response, config).first).dig(0, -1)
+        got = Response::Factory.response(response, config).first
+
+        case got
+        when Hash
+          Array(got).dig(0, -1) # get a value of a first key for JSON format
+        when Array
+          got[0] # for CSV format
+        else
+          got # for RowBinary format
+        end
       end
 
       def select_one(sql)

data/lib/click_house/middleware/logging.rb

@@ -5,36 +5,32 @@ module ClickHouse
     class Logging < Faraday::Middleware
       Faraday::Response.register_middleware self => self
 
-      SUMMARY_HEADER = 'x-clickhouse-summary'
+      EMPTY = ''
+      GET = :get
 
-      attr_reader :logger, :starting, :body
+      attr_reader :logger, :starting
 
       def initialize(app = nil, logger:)
         @logger = logger
         super(app)
       end
 
-      def call(environment)
+      def call(env)
         @starting = timestamp
-        @body = environment.body if log_body?
-        @app.call(environment).on_complete(&method(:on_complete))
-      end
-
-      private
-
-      def log_body?
-        logger.level == Logger::DEBUG
+        super
       end
 
       # rubocop:disable Layout/LineLength
       def on_complete(env)
-        summary = extract_summary(env.response_headers)
-        logger.info("\e[1mSQL (#{duration_stats_log(env.body)})\e[0m #{query(env)};")
-        logger.debug(body) if body
-        logger.info("\e[1mRead: #{summary.fetch(:read_rows)} rows, #{summary.fetch(:read_bytes)}. Written: #{summary.fetch(:written_rows)} rows, #{summary.fetch(:written_bytes)}\e[0m")
+        summary = SummaryMiddleware.extract(env)
+        logger.info("\e[1mSQL (#{duration_stats_log(summary)})\e[0m #{query(env)};")
+        logger.debug(env.request_body) if log_body?(env)
+        logger.info("\e[1mRead: #{summary.read_rows} rows, #{summary.read_bytes_pretty}. Written: #{summary.written_rows} rows, #{summary.written_bytes_pretty}\e[0m")
       end
       # rubocop:enable Layout/LineLength
 
+      private
+
       def duration
         timestamp - starting
       end
@@ -43,37 +39,25 @@ module ClickHouse
         Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end
 
-      def query_in_body?(env)
-        env.method == :get
+      # @return [Boolean]
+      def log_body?(env)
+        return unless logger.debug?
+        return if env.method == GET # GET queries logs body as a statement
+        return if env.request_body.nil? || env.request_body == EMPTY
+
+        true
       end
 
       def query(env)
-        if query_in_body?(env)
-          body
+        if env.method == GET
+          env.request_body
         else
-          CGI.parse(env.url.query.to_s).dig('query', 0) || '[NO QUERY]'
+          String(CGI.parse(env.url.query.to_s).dig('query', 0) || '[NO QUERY]').chomp
         end
       end
 
-      def duration_stats_log(body)
-        elapsed = duration
-        clickhouse_elapsed = body['statistics'].fetch('elapsed') if body.is_a?(Hash) && body.key?('statistics')
-
-        [
-          "Total: #{Util::Pretty.measure(elapsed * 1000)}",
-          ("CH: #{Util::Pretty.measure(clickhouse_elapsed * 1000)}" if clickhouse_elapsed)
-        ].compact.join(', ')
-      end
-
-      def extract_summary(headers)
-        JSON.parse(headers.fetch('x-clickhouse-summary', '{}')).tap do |summary|
-          summary[:read_rows] = summary['read_rows']
-          summary[:read_bytes] = Util::Pretty.size(summary['read_bytes'].to_i)
-          summary[:written_rows] = summary['written_rows']
-          summary[:written_bytes] = Util::Pretty.size(summary['written_bytes'].to_i)
-        end
-      rescue JSON::ParserError
-        {}
+      def duration_stats_log(summary)
+        "Total: #{Util::Pretty.measure(duration * 1000)}, CH: #{summary.elapsed_pretty}"
       end
     end
   end

data/lib/click_house/middleware/response_base.rb

@@ -5,12 +5,12 @@ module ClickHouse
     class ResponseBase < Faraday::Middleware
       CONTENT_TYPE_HEADER = 'content-type'
 
-      attr_reader :parser_options
+      attr_reader :options
       attr_reader :content_type
 
-      def initialize(app = nil, parser_options: {}, content_type: nil, preserve_raw: false)
+      def initialize(app = nil, options: {}, content_type: nil, preserve_raw: false)
         super(app)
-        @parser_options = parser_options
+        @options = options
         @content_type = content_type
         @preserve_raw = preserve_raw
         on_setup
@@ -32,7 +32,7 @@ module ClickHouse
 
       # @return [Config]
       def config
-        parser_options.fetch(:config)
+        options.fetch(:config)
       end
 
       private

data/lib/click_house/middleware/summary_middleware.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Middleware
+    class SummaryMiddleware < ResponseBase
+      Faraday::Response.register_middleware self => self
+
+      KEY = :summary
+
+      # @param env [Faraday::Env]
+      # @return [Response::Summary]
+      def self.extract(env)
+        env.custom_members.fetch(KEY)
+      end
+
+      # @param env [Faraday::Env]
+      def on_complete(env)
+        env.custom_members[KEY] = Response::Summary.new(
+          config,
+          headers: env.response_headers,
+          body: env.body.is_a?(Hash) ? env.body : {}
+        )
+      end
+    end
+  end
+end

data/lib/click_house/middleware.rb

@@ -3,6 +3,7 @@
 module ClickHouse
   module Middleware
     autoload :ResponseBase, 'click_house/middleware/response_base'
+    autoload :SummaryMiddleware, 'click_house/middleware/summary_middleware'
     autoload :Logging, 'click_house/middleware/logging'
     autoload :ParseCsv, 'click_house/middleware/parse_csv'
     autoload :ParseJsonOj, 'click_house/middleware/parse_json_oj'

data/lib/click_house/response/factory.rb

@@ -5,38 +5,46 @@ module ClickHouse
     class Factory
       KEY_META = 'meta'
       KEY_DATA = 'data'
-      KEY_TOTALS = 'totals'
-      KEY_STATISTICS = 'statistics'
-      KEY_ROWS_BEFORE_LIMIT_AT_LEAST = 'rows_before_limit_at_least'
 
-      # @return [Nil], ResultSet]
+      # @return [ResultSet]
       # @params faraday [Faraday::Response]
       # @params config [Config]
       def self.response(faraday, config)
         body = faraday.body
 
-        return body unless body.is_a?(Hash)
-        return body unless body.key?(config.key(KEY_META)) && body.key?(config.key(KEY_DATA))
+        # wrap to be able to use connection#select_one, connection#select_value
+        # with other formats like binary
+        return raw(faraday, config) unless body.is_a?(Hash)
+        return raw(faraday, config) unless body.key?(config.key(KEY_META)) && body.key?(config.key(KEY_DATA))
 
         ResultSet.new(
           config: config,
           meta: body.fetch(config.key(KEY_META)),
           data: body.fetch(config.key(KEY_DATA)),
-          totals: body[config.key(KEY_TOTALS)],
-          statistics: body[config.key(KEY_STATISTICS)],
-          rows_before_limit_at_least: body[config.key(KEY_ROWS_BEFORE_LIMIT_AT_LEAST)]
+          summary: Middleware::SummaryMiddleware.extract(faraday.env)
         )
       end
 
-      # @return [Response::Execution]
+      # @return [ResultSet]
+      # Rae ResultSet (without type casting)
+      def self.raw(faraday, config)
+        ResultSet.raw(
+          config: config,
+          data: Util.array(faraday.body),
+          summary: Middleware::SummaryMiddleware.extract(faraday.env)
+        )
+      end
+
+      # Result of execution
+      # @return [Response::Summary]
       # @params faraday [Faraday::Response]
       def self.exec(faraday)
-        Execution.new(headers: faraday.headers)
+        Middleware::SummaryMiddleware.extract(faraday.env)
       end
 
-      # @return [Response::Execution]
-      def self.empty_exec
-        Execution.new
+      # @return [Response::Summary]
+      def self.empty_exec(config)
+        Summary.new(config)
       end
     end
   end

data/lib/click_house/response/result_set.rb

@@ -13,24 +13,31 @@ module ClickHouse
                      :inspect, :each, :fetch, :length, :count, :size,
                      :first, :last, :[], :to_h
 
-      attr_reader :config, :meta, :data, :totals, :statistics, :rows_before_limit_at_least
+      def_delegators :summary,
+                     :statistics, :headers,
+                     :totals, :rows_before_limit_at_least
+
+      attr_reader :config, :meta, :data, :summary
+
+      class << self
+        # @param config [Config]
+        # @return [ResultSet]
+        def raw(config:, data:, summary:)
+          new(config: config, data: data, to_a: data, meta: [], summary: summary)
+        end
+      end
 
       # @param config [Config]
       # @param meta [Array]
       # @param data [Array]
-      # @param totals [Array|Hash|NilClass] Support for 'GROUP BY WITH TOTALS' modifier
-      #   https://clickhouse.tech/docs/en/sql-reference/statements/select/group-by/#with-totals-modifier
-      #   Hash in JSON format and Array in JSONCompact
-      # rubocop:disable Metrics/ParameterLists
-      def initialize(config:, meta:, data:, totals: nil, statistics: nil, rows_before_limit_at_least: nil)
+      # @param summary [Response::Summary]
+      def initialize(config:, meta:, data:, summary:, to_a: nil)
         @config = config
         @meta = meta
         @data = data
-        @totals = totals
-        @rows_before_limit_at_least = rows_before_limit_at_least
-        @statistics = Hash(statistics)
+        @summary = summary
+        @to_a = to_a
       end
-      # rubocop:enable Metrics/ParameterLists
 
       # @return [Array, Hash]
       # @param data [Array, Hash]

data/lib/click_house/response/summary.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Response
+    class Summary
+      SUMMARY_HEADER = 'x-clickhouse-summary'
+      KEY_TOTALS = 'totals'
+      KEY_STATISTICS = 'statistics'
+      KEY_ROWS_BEFORE_LIMIT_AT_LEAST = 'rows_before_limit_at_least'
+      KEY_STAT_ELAPSED = 'elapsed'
+
+      attr_reader :config,
+                  :headers,
+                  :summary,
+                  # {:elapsed=>0.387287e-3, :rows_read=>0, :bytes_read=>0}}
+                  :statistics,
+                  :totals,
+                  :rows_before_limit_at_least
+
+      # @param config [Config]
+      # @param headers [Faraday::Utils::Headers]
+      # @param body [Hash]
+      # TOTALS [Array|Hash|NilClass] Support for 'GROUP BY WITH TOTALS' modifier
+      #   https://clickhouse.tech/docs/en/sql-reference/statements/select/group-by/#with-totals-modifier
+      #   Hash in JSON format and Array in JSONCompact
+      def initialize(config, headers: Faraday::Utils::Headers.new, body: {})
+        @headers = headers
+        @config = config
+        @statistics = body.fetch(config.key(KEY_STATISTICS), {})
+        @totals = body[config.key(KEY_TOTALS)]
+        @rows_before_limit_at_least = body[config.key(KEY_ROWS_BEFORE_LIMIT_AT_LEAST)]
+        @summary = parse_summary(headers[SUMMARY_HEADER])
+      end
+
+      # @return [Integer]
+      def read_rows
+        summary[config.key('read_rows')].to_i
+      end
+
+      # @return [Integer]
+      def read_bytes
+        summary[config.key('read_bytes')].to_i
+      end
+
+      # @return [String]
+      def read_bytes_pretty
+        Util::Pretty.size(read_bytes)
+      end
+
+      # @return [Integer]
+      def written_rows
+        summary[config.key('written_rows')].to_i
+      end
+
+      # @return [Integer]
+      def written_bytes
+        summary[config.key('written_bytes')].to_i
+      end
+
+      # @return [String]
+      def written_bytes_pretty
+        Util::Pretty.size(written_bytes)
+      end
+
+      # @return [Integer]
+      def total_rows_to_read
+        summary[config.key('total_rows_to_read')].to_i
+      end
+
+      # @return [Integer]
+      def result_rows
+        summary[config.key('result_rows')].to_i
+      end
+
+      # @return [Integer]
+      def result_bytes
+        summary[config.key('result_bytes')].to_i
+      end
+
+      # @return [Float]
+      def elapsed
+        statistics[config.key(KEY_STAT_ELAPSED)].to_f
+      end
+
+      # @return [String]
+      def elapsed_pretty
+        Util::Pretty.measure(elapsed * 1000)
+      end
+
+      private
+
+      # @return [Hash]
+      # {
+      #   "read_rows" => "1",
+      #   "read_bytes" => "23",
+      #   "written_rows" => "1",
+      #   "written_bytes" => "23",
+      #   "total_rows_to_read" => "0",
+      #   "result_rows" => "1",
+      #   "result_bytes" => "23",
+      # }
+      def parse_summary(value)
+        return {} if value.nil? || value.empty?
+
+        JSON.parse(value)
+      end
+    end
+  end
+end

data/lib/click_house/response.rb

@@ -4,6 +4,6 @@ module ClickHouse
   module Response
     autoload :Factory, 'click_house/response/factory'
     autoload :ResultSet, 'click_house/response/result_set'
-    autoload :Execution, 'click_house/response/execution'
+    autoload :Summary, 'click_house/response/summary'
   end
 end

data/lib/click_house/util.rb

@@ -4,5 +4,12 @@ module ClickHouse
   module Util
     autoload :Statement, 'click_house/util/statement'
     autoload :Pretty, 'click_house/util/pretty'
+
+    module_function
+
+    # wraps
+    def array(input)
+      input.is_a?(Array) ? input : [input]
+    end
   end
 end

data/lib/click_house/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ClickHouse
-  VERSION = '2.1.0'
+  VERSION = '2.1.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: click_house
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.1.1
 platform: ruby
 authors:
 - Aliaksandr Shylau
@@ -200,10 +200,11 @@ files:
 - lib/click_house/middleware/parse_json_oj.rb
 - lib/click_house/middleware/raise_error.rb
 - lib/click_house/middleware/response_base.rb
+- lib/click_house/middleware/summary_middleware.rb
 - lib/click_house/response.rb
-- lib/click_house/response/execution.rb
 - lib/click_house/response/factory.rb
 - lib/click_house/response/result_set.rb
+- lib/click_house/response/summary.rb
 - lib/click_house/serializer.rb
 - lib/click_house/serializer/base.rb
 - lib/click_house/serializer/json_oj_serializer.rb

data/lib/click_house/response/execution.rb

@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-module ClickHouse
-  module Response
-    class Execution
-      SUMMARY_HEADER = 'x-clickhouse-summary'
-
-      attr_reader :headers, :summary
-
-      # @param headers [Faraday::Utils::Headers]
-      def initialize(headers: Faraday::Utils::Headers.new)
-        @headers = headers
-        @summary = parse_summary(headers[SUMMARY_HEADER])
-      end
-
-      # @return [Integer]
-      def read_rows
-        summary['read_rows'].to_i
-      end
-
-      # @return [Integer]
-      def read_bytes
-        summary['read_bytes'].to_i
-      end
-
-      # @return [Integer]
-      def written_rows
-        summary['written_rows'].to_i
-      end
-
-      # @return [Integer]
-      def written_bytes
-        summary['written_bytes'].to_i
-      end
-
-      # @return [Integer]
-      def total_rows_to_read
-        summary['total_rows_to_read'].to_i
-      end
-
-      # @return [Integer]
-      def result_rows
-        summary['result_rows'].to_i
-      end
-
-      # @return [Integer]
-      def result_bytes
-        summary['result_bytes'].to_i
-      end
-
-      private
-
-      # @return [Hash]
-      # {
-      #   "read_rows" => "1",
-      #   "read_bytes" => "23",
-      #   "written_rows" => "1",
-      #   "written_bytes" => "23",
-      #   "total_rows_to_read" => "0",
-      #   "result_rows" => "1",
-      #   "result_bytes" => "23",
-      # }
-      def parse_summary(value)
-        return {} if value.nil? || value.empty?
-
-        JSON.parse(value)
-      end
-    end
-  end
-end