click_house 2.0.2 → 2.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c2d62b43147b81ad3551c48c79388a949307d03491c1d87965d25ae9818e0a86
-  data.tar.gz: ad6e79c3a974e4e39889d93dcabd94d3dc8f2bced9b739449f741bd1c8c83038
+  metadata.gz: 364117bb31fd3478fcf00d981b61205aa9fb864abbfeb9f9bdf253e53a0917fa
+  data.tar.gz: d0b0d7a21175a9c2e044a88351ddd6d107b1875769ae737d3e547e41e01d93d1
 SHA512:
-  metadata.gz: 9035db18ecc851ab6de8f9ae88d434dfff07f2da5e23eac4cf3c091fe699871c64f0c9c05a1637730f44dfd5fef4eca60f6cd94f613360113152461f7e4dabe8
-  data.tar.gz: bb93033b9ffc82a7003af0affea015ad2a84292a86a50c46ea6cecfa232b22d172708dc1633f896fd29eea373270eb464baa70e02df8d036195d0aeb2f88f29c
+  metadata.gz: 481ba8166f2300302c1a0836e336097e97a25015fbe5a5cf8e8fb5912b2170d50a2a1bc25178deb8b4a4b17d63e5638d0200bbca5a84c6a237de694ecf5f4d76
+  data.tar.gz: ba051b743929517452507d54f98de6d57de724b4400d1cf9db02666a48c5b00d4dbd00a3ffab6b5f2f9c3a7db78c15f84ae36523e2ee76946526fe131722ee05

data/CHANGELOG.md

@@ -1,3 +1,40 @@
+# 2.1.1
+* Fix logging with symbolized keys JSON
+* Unknown formats return raw `Response::ResultSelt` like regular JSON query
+
+# 2.1.0
+* `ClickHouse.connection.insert` now returns `ClickHouse::Response::Summary` object
+  with methods `headers`, `summary`, `written_rows`, `written_bytes`, etc... 
+* `ClickHouse.connection.insert(columns: ["id"], values: [1])` now uses `JSONCompactEachRow` by default
+  (to increase JSON serialization speed)
+* Methods `insert_rows` and `insert_compact` added to `connection`
+* Added ability to pass object directly to insert like:
+  `ClickHouse.connection.insert("table", {id: 1})` or
+  `ClickHouse.connection.insert("table", [{id: 1})]` (for ruby < 3.0 use `ClickHouse.connection.insert("table", [{id: 1}], {})`)
+* 🔥 Added config option `json_serializer` (one of `ClickHouse::Serializer::JsonSerializer`, `ClickHouse::Serializer::JsonOjSerializer`)
+* 🔥 Added config option `symbolize_keys`
+* 🔥 Added type serialization for INSERT statements, example below:
+
+```sql
+CREATE TABLE assets(visible Boolean, tags Array(Nullable(String))) ENGINE Memory
+```
+
+```ruby
+# cache table schema in a class variable
+@schema = ClickHouse.connection.table_schema('assets')
+
+# Json each row
+ClickHouse.connection.insert('assets', @schema.serialize({'visible' => true, 'tags' => ['ruby']}))
+
+# Json compact
+ClickHouse.connection.insert('assets', columns: %w[visible tags]) do |buffer|
+  buffer << [
+    @schema.serialize_column("visible", true),
+    @schema.serialize_column("tags", ['ruby']),
+  ]
+end
+```
+
 # 2.0.0
 * Fixed `Bigdecimal` casting with high precision
 * Added nested `type casting like Array(Array(Array(Nullable(T))))`
@@ -5,6 +42,7 @@
 * Added `Tuple(T1, T2)` support
 * Added support for `Faraday` v1 and v2
 * Added support for `Oj` parser
+* Time types return `Time` class instead of `DateTime` for now
 
 # 1.6.3
 * [PR](https://github.com/shlima/click_house/pull/38) Add option format for insert

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    click_house (2.0.2)
+    click_house (2.1.1)
       activesupport
       faraday (>= 1.7, < 3)
 

data/Gemfile_faraday1

@@ -5,7 +5,7 @@ source 'https://rubygems.org'
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
 # lock faraday to v1
-gem 'faraday_middleware'
+gem 'faraday', '< 2'
 
 # Specify your gem's dependencies in click_house.gemspec
 gemspec

data/Gemfile_faraday1.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    click_house (2.0.1)
+    click_house (2.1.1)
       activesupport
       faraday (>= 1.7, < 3)
 
@@ -40,8 +40,6 @@ GEM
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
     faraday-retry (1.0.3)
-    faraday_middleware (1.2.0)
-      faraday (~> 1.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
     json (2.6.2)
@@ -94,12 +92,13 @@ GEM
     unicode-display_width (2.3.0)
 
 PLATFORMS
+  ruby
   x86_64-darwin-21
 
 DEPENDENCIES
   bundler
   click_house!
-  faraday_middleware
+  faraday (< 2)
   oj
   pry
   rake

data/Gemfile_faraday2.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    click_house (1.7.0)
+    click_house (2.1.1)
       activesupport
       faraday (>= 1.7, < 3)
 

data/README.md

@@ -52,6 +52,8 @@ ClickHouse.config do |config|
   config.timeout = 60
   config.open_timeout = 3
   config.ssl_verify = false
+  # set to true to symbolize keys for SELECT and INSERT statements (type casting)
+  config.symbolize_keys = false
   config.headers = {}
 
   # or provide connection options separately
@@ -66,10 +68,15 @@ ClickHouse.config do |config|
   # if you want to add settings to all queries
   config.global_params = { mutations_sync: 1 }
   
-  # choose a ruby JSON parser
+  # choose a ruby JSON parser (default one)
   config.json_parser = ClickHouse::Middleware::ParseJson
   # or Oj parser
   config.json_parser = ClickHouse::Middleware::ParseJsonOj
+
+  # JSON.dump (default one)
+  config.json_serializer = ClickHouse::Serializer::JsonSerializer
+  # or Oj.dump
+  config.json_serializer = ClickHouse::Serializer::JsonOjSerializer
 end
 ```
 
@@ -131,12 +138,16 @@ Select all type-casted result set
 @result = ClickHouse.connection.select_all('SELECT * FROM visits')
 
 # all enumerable methods are delegated like #each, #map, #select etc
+# results of #to_a is type casted
 @result.to_a #=> [{"date"=>#<Date: 2000-01-01>, "id"=>1}]
 
 # you can access raw data
 @result.meta #=> [{"name"=>"date", "type"=>"Date"}, {"name"=>"id", "type"=>"UInt32"}]
 @result.data #=> [{"date"=>"2000-01-01", "id"=>1}, {"date"=>"2000-01-02", "id"=>2}]
 @result.statistics #=> {"elapsed"=>0.0002271, "rows_read"=>2, "bytes_read"=>12}
+@result.summary #=> ClickHouse::Response::Summary
+@result.headers #=> {"x-clickhouse-query-id"=>"9bf5f604-31fc-4eff-a4b5-277f2c71d199"}
+@result.types #=> [Hash<String|Symbol, ClickHouse::Ast::Statement>]
 ```
 
 ### Select Value
@@ -195,7 +206,8 @@ response.body #=> "\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000"
 
 ## Insert
 
-When column names and values are transferred separately
+When column names and values are transferred separately, data sends to the server 
+using `JSONCompactEachRow` format by default.
 
 ```ruby
 ClickHouse.connection.insert('table', columns: %i[id name]) do |buffer|
@@ -203,23 +215,42 @@ ClickHouse.connection.insert('table', columns: %i[id name]) do |buffer|
   buffer << [2, 'Venus']
 end
 
+# or
 ClickHouse.connection.insert('table', columns: %i[id name], values: [[1, 'Mercury'], [2, 'Venus']])
-#=> true
 ```
 
-When rows are passed as a hash
-
+When rows are passed as an Array or a Hash, data sends to the server
+using `JSONEachRow` format by default.
 
 ```ruby
-ClickHouse.connection.insert('table', values: [{ name: 'Sun', id: 1 }, { name: 'Moon', id: 2 }])
+ClickHouse.connection.insert('table', [{ name: 'Sun', id: 1 }, { name: 'Moon', id: 2 }])
+
+# or
+ClickHouse.connection.insert('table', { name: 'Sun', id: 1 })
+
+# for ruby < 3.0 provide an extra argument
+ClickHouse.connection.insert('table', { name: 'Sun', id: 1 }, {})
 
+# or
 ClickHouse.connection.insert('table') do |buffer|
   buffer << { name: 'Sun', id: 1 }
   buffer << { name: 'Moon', id: 2 }
 end
-#=> true
 ```
 
+Sometimes it's needed to use other format than `JSONEachRow` For example if you want to send BigDecimal's 
+you could use `JSONStringsEachRow` format so string representation of `BigDecimal` will be parsed:
+
+```ruby
+ClickHouse.connection.insert('table', { name: 'Sun', id: '1' }, format: 'JSONStringsEachRow')
+# or
+ClickHouse.connection.insert_rows('table', { name: 'Sun', id: '1' }, format: 'JSONStringsEachRow')
+# or
+ClickHouse.connection.insert_compact('table', columns: %w[name id], values: %w[Sun 1], format: 'JSONCompactStringsEachRow')
+```
+
+See the [type casting](#type-casting) section to insert the data in a proper way.
+
 ## Create a table
 ### Create table using DSL
 
@@ -326,42 +357,35 @@ end
 ## Type casting
 
 By default gem provides all necessary type casting, but you may overwrite or define
-your own logic
+your own logic. if you need to redefine all built-in types with your implementation,
+just clear the default type system:
 
 ```ruby
-class DateType
-  def cast(value)
-    Date.parse(value)
-  end
-
-  def serialize(value)
-    value.strftime('%Y-%m-%d')
-  end
-end
-
-ClickHouse.add_type('Date', DateType.new)
+ClickHouse.types.clear
+ClickHouse.types # => {}
+ClickHouse.types.default #=> #<ClickHouse::Type::UndefinedType>
 ```
 
-If native type supports arguments, define *String* type with `%s`
-argument and *Numeric* type with `%d` argument:
+Type casting works automatically when fetching data, when inserting data, you must serialize the types yourself
 
-```ruby
-class DateTimeType
-  def cast(value, time_zone)
-    Time.parse("#{value} #{time_zone}")
-  end
-end
-
-ClickHouse.add_type('DateTime(%s)', DateTimeType.new)
+```sql
+CREATE TABLE assets(visible Boolean, tags Array(Nullable(String))) ENGINE Memory
 ```
 
-if you need to redefine all built-in types with your implementation,
-just clear the default type system:
-
 ```ruby
-ClickHouse.types.clear
-ClickHouse.types # => {}
-ClickHouse.types.default #=> #<ClickHouse::Type::UndefinedType:0x00007fc1cfabd630>
+# cache table schema in a class variable
+@schema = ClickHouse.connection.table_schema('assets')
+
+# Json each row
+ClickHouse.connection.insert('assets', @schema.serialize({'visible' => true, 'tags' => ['ruby']}))
+
+# Json compact
+ClickHouse.connection.insert('assets', columns: %w[visible tags]) do |buffer|
+  buffer << [
+    @schema.serialize_column("visible", true),
+    @schema.serialize_column("tags", ['ruby']),
+  ]
+end
 ```
 
 ## Using with a connection pool

data/lib/click_house/benchmark/map_join.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'benchmark'
+require 'stringio'
+
+INPUT = Array.new(5_000_000, 'foo bar')
+
+Benchmark.bm do |x|
+  x.report('map.join') do
+    INPUT.map(&:to_s).join("\n")
+  end
+
+  x.report('StringIO') do
+    out = StringIO.new
+    INPUT.each do |value|
+      out << "#{value}\n"
+    end
+    out.string
+  end
+end

data/lib/click_house/config.rb

@@ -18,18 +18,21 @@ module ClickHouse
       headers: {},
       global_params: {},
       json_parser: ClickHouse::Middleware::ParseJson,
+      json_serializer: ClickHouse::Serializer::JsonSerializer,
+      oj_dump_options: {
+        mode: :compat # to be able to dump improper JSON like {1 => 2}
+      },
       oj_load_options: {
         mode: :custom,
         allow_blank: true,
         bigdecimal_as_decimal: false, # dump BigDecimal as a String
         bigdecimal_load: :bigdecimal, # convert all decimal numbers to BigDecimal
-        empty_string: false,
-        second_precision: 6,
-        time_format: :ruby,
       },
       json_load_options: {
         decimal_class: BigDecimal,
       },
+      # should be after json load options
+      symbolize_keys: false,
     }.freeze
 
     attr_accessor :adapter
@@ -49,6 +52,9 @@ module ClickHouse
     attr_accessor :oj_load_options
     attr_accessor :json_load_options
     attr_accessor :json_parser # response middleware
+    attr_accessor :oj_dump_options
+    attr_accessor :json_serializer # [ClickHouse::Serializer::Base]
+    attr_accessor :symbolize_keys # [NilClass, Boolean]
 
     def initialize(params = {})
       assign(DEFAULTS.merge(params))
@@ -77,5 +83,25 @@ module ClickHouse
     def null_logger
       @null_logger ||= Logger.new(IO::NULL)
     end
+
+    # @param klass [ClickHouse::Serializer::Base]
+    def json_serializer=(klass)
+      @json_serializer = klass.new(self)
+    end
+
+    def symbolize_keys=(value)
+      bool = value ? true : false
+
+      # merge to be able to clone a config
+      # prevent overriding default values
+      self.oj_load_options = oj_load_options.merge(symbol_keys: bool)
+      self.json_load_options = json_load_options.merge(symbolize_names: bool)
+      @symbolize_keys = bool
+    end
+
+    # @param name [Symbol, String]
+    def key(name)
+      symbolize_keys ? name.to_sym : name.to_s
+    end
   end
 end

data/lib/click_house/connection.rb

@@ -65,8 +65,9 @@ module ClickHouse
 
         conn.response Middleware::RaiseError
         conn.response Middleware::Logging, logger: config.logger!
-        conn.response config.json_parser, content_type: %r{application/json}, parser_options: { config: config }
-        conn.response Middleware::ParseCsv, content_type: %r{text/csv}, parser_options: { config: config }
+        conn.response Middleware::SummaryMiddleware, options: { config: config } # should be after logger
+        conn.response config.json_parser, content_type: %r{application/json}, options: { config: config }
+        conn.response Middleware::ParseCsv, content_type: %r{text/csv}, options: { config: config }
         conn.adapter config.adapter
       end
     end

data/lib/click_house/errors.rb

@@ -5,4 +5,5 @@ module ClickHouse
   NetworkException = Class.new(Error)
   DbException = Class.new(Error)
   StatementException = Class.new(Error)
+  SerializeError = Class.new(Error)
 end

data/lib/click_house/extend/connection_inserting.rb

@@ -3,7 +3,8 @@
 module ClickHouse
   module Extend
     module ConnectionInserting
-      EMPTY_INSERT = true
+      DEFAULT_JSON_EACH_ROW_FORMAT = 'JSONEachRow'
+      DEFAULT_JSON_COMPACT_EACH_ROW_FORMAT = 'JSONCompactEachRow'
 
       # @return [Boolean]
       #
@@ -13,20 +14,71 @@ module ClickHouse
       #   buffer << ['Moon', 2]
       # end
       #
-      # == Example with a param
-      # subject.insert('rspec', values: [{ name: 'Sun', id: 1 }, { name: 'Moon', id: 2 }], format: 'JSONStringsEachRow')
-      def insert(table, columns: [], values: [], format: 'JSONEachRow')
-        yield(values) if block_given?
+      # @return [Response::Execution]
+      # @param body [Array, Hash]
+      # rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+      def insert(table, body = [], **opts)
+        # In Ruby < 3.0, if the last argument is a hash, and the method being called
+        # accepts keyword arguments, then it is always converted to keyword arguments.
+        columns = opts.fetch(:columns, [])
+        values =  opts.fetch(:values, [])
+        format = opts.fetch(:format, nil)
+
+        yield(body) if block_given?
 
-        body = if columns.empty?
-          values.map(&:to_json)
+        # values: [{id: 1}]
+        if values.any? && columns.empty?
+          return insert_rows(table, values, format: format)
+        end
+
+        # body: [{id: 1}]
+        if body.any? && columns.empty?
+          return insert_rows(table, body, format: format)
+        end
+
+        # body: [1], columns: ["id"]
+        if body.any? && columns.any?
+          return insert_compact(table, columns: columns, values: body, format: format)
+        end
+
+        # columns: ["id"], values: [[1]]
+        if columns.any? && values.any?
+          return insert_compact(table, columns: columns, values: values, format: format)
+        end
+
+        Response::Factory.empty_exec(config)
+      end
+      # rubocop:enable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+
+      # @param table [String]
+      # @param body [Array, Hash]
+      # @param format [String]
+      # @return [Response::Execution]
+      #
+      # Sometimes it's needed to use other format than JSONEachRow
+      # For example if you want to send BigDecimal's you could use
+      # JSONStringsEachRow format so string representation of BigDecimal will be parsed
+      def insert_rows(table, body, format: nil)
+        format ||= DEFAULT_JSON_EACH_ROW_FORMAT
+
+        case body
+        when Hash
+          Response::Factory.exec(execute("INSERT INTO #{table} FORMAT #{format}", config.json_serializer.dump(body)))
+        when Array
+          Response::Factory.exec(execute("INSERT INTO #{table} FORMAT #{format}", config.json_serializer.dump_each_row(body)))
         else
-          values.map { |value_row| columns.zip(value_row).to_h.to_json }
+          raise ArgumentError, "unknown body class <#{body.class}>"
         end
+      end
 
-        return EMPTY_INSERT if values.empty?
+      # @return [Response::Execution]
+      def insert_compact(table, columns: [], values: [], format: nil)
+        format ||= DEFAULT_JSON_COMPACT_EACH_ROW_FORMAT
+
+        yield(values) if block_given?
 
-        execute("INSERT INTO #{table} FORMAT #{format}", body.join("\n")).success?
+        response = execute("INSERT INTO #{table} (#{columns.join(',')}) FORMAT #{format}", config.json_serializer.dump_each_row(values))
+        Response::Factory.exec(response)
       end
     end
   end

data/lib/click_house/extend/connection_selective.rb

@@ -6,17 +6,26 @@ module ClickHouse
       # @return [ResultSet]
       def select_all(sql)
         response = get(body: sql, query: { default_format: 'JSON' })
-        Response::Factory[response]
+        Response::Factory.response(response, config)
       end
 
       def select_value(sql)
         response = get(body: sql, query: { default_format: 'JSON' })
-        Array(Response::Factory[response].first).dig(0, -1)
+        got = Response::Factory.response(response, config).first
+
+        case got
+        when Hash
+          Array(got).dig(0, -1) # get a value of a first key for JSON format
+        when Array
+          got[0] # for CSV format
+        else
+          got # for RowBinary format
+        end
       end
 
       def select_one(sql)
         response = get(body: sql, query: { default_format: 'JSON' })
-        Response::Factory[response].first
+        Response::Factory.response(response, config).first
       end
     end
   end

data/lib/click_house/extend/connection_table.rb

@@ -10,7 +10,12 @@ module ClickHouse
 
       # @return [ResultSet]
       def describe_table(name)
-        Response::Factory[execute("DESCRIBE TABLE #{name} FORMAT JSON")]
+        Response::Factory.response(execute("DESCRIBE TABLE #{name} FORMAT JSON"), config)
+      end
+
+      # @return [ResultSet]
+      def table_schema(name)
+        Response::Factory.response(execute("SELECT * FROM #{name} WHERE 1=0 FORMAT JSON"), config)
       end
 
       # @return [Boolean]

data/lib/click_house/middleware/logging.rb

@@ -5,36 +5,32 @@ module ClickHouse
     class Logging < Faraday::Middleware
       Faraday::Response.register_middleware self => self
 
-      SUMMARY_HEADER = 'x-clickhouse-summary'
+      EMPTY = ''
+      GET = :get
 
-      attr_reader :logger, :starting, :body
+      attr_reader :logger, :starting
 
       def initialize(app = nil, logger:)
         @logger = logger
         super(app)
       end
 
-      def call(environment)
+      def call(env)
         @starting = timestamp
-        @body = environment.body if log_body?
-        @app.call(environment).on_complete(&method(:on_complete))
-      end
-
-      private
-
-      def log_body?
-        logger.level == Logger::DEBUG
+        super
       end
 
       # rubocop:disable Layout/LineLength
       def on_complete(env)
-        summary = extract_summary(env.response_headers)
-        logger.info("\e[1mSQL (#{duration_stats_log(env.body)})\e[0m #{query(env)};")
-        logger.debug(body) if body
-        logger.info("\e[1mRead: #{summary.fetch(:read_rows)} rows, #{summary.fetch(:read_bytes)}. Written: #{summary.fetch(:written_rows)} rows, #{summary.fetch(:written_bytes)}\e[0m")
+        summary = SummaryMiddleware.extract(env)
+        logger.info("\e[1mSQL (#{duration_stats_log(summary)})\e[0m #{query(env)};")
+        logger.debug(env.request_body) if log_body?(env)
+        logger.info("\e[1mRead: #{summary.read_rows} rows, #{summary.read_bytes_pretty}. Written: #{summary.written_rows} rows, #{summary.written_bytes_pretty}\e[0m")
       end
       # rubocop:enable Layout/LineLength
 
+      private
+
       def duration
         timestamp - starting
       end
@@ -43,37 +39,25 @@ module ClickHouse
         Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end
 
-      def query_in_body?(env)
-        env.method == :get
+      # @return [Boolean]
+      def log_body?(env)
+        return unless logger.debug?
+        return if env.method == GET # GET queries logs body as a statement
+        return if env.request_body.nil? || env.request_body == EMPTY
+
+        true
       end
 
       def query(env)
-        if query_in_body?(env)
-          body
+        if env.method == GET
+          env.request_body
         else
-          CGI.parse(env.url.query.to_s).dig('query', 0) || '[NO QUERY]'
+          String(CGI.parse(env.url.query.to_s).dig('query', 0) || '[NO QUERY]').chomp
         end
       end
 
-      def duration_stats_log(body)
-        elapsed = duration
-        clickhouse_elapsed = body['statistics'].fetch('elapsed') if body.is_a?(Hash) && body.key?('statistics')
-
-        [
-          "Total: #{Util::Pretty.measure(elapsed * 1000)}",
-          ("CH: #{Util::Pretty.measure(clickhouse_elapsed * 1000)}" if clickhouse_elapsed)
-        ].compact.join(', ')
-      end
-
-      def extract_summary(headers)
-        JSON.parse(headers.fetch('x-clickhouse-summary', '{}')).tap do |summary|
-          summary[:read_rows] = summary['read_rows']
-          summary[:read_bytes] = Util::Pretty.size(summary['read_bytes'].to_i)
-          summary[:written_rows] = summary['written_rows']
-          summary[:written_bytes] = Util::Pretty.size(summary['written_bytes'].to_i)
-        end
-      rescue JSON::ParserError
-        {}
+      def duration_stats_log(summary)
+        "Total: #{Util::Pretty.measure(duration * 1000)}, CH: #{summary.elapsed_pretty}"
       end
     end
   end

data/lib/click_house/middleware/response_base.rb

@@ -5,12 +5,12 @@ module ClickHouse
     class ResponseBase < Faraday::Middleware
       CONTENT_TYPE_HEADER = 'content-type'
 
-      attr_reader :parser_options
+      attr_reader :options
       attr_reader :content_type
 
-      def initialize(app = nil, parser_options: {}, content_type: nil, preserve_raw: false)
+      def initialize(app = nil, options: {}, content_type: nil, preserve_raw: false)
         super(app)
-        @parser_options = parser_options
+        @options = options
         @content_type = content_type
         @preserve_raw = preserve_raw
         on_setup
@@ -32,7 +32,7 @@ module ClickHouse
 
       # @return [Config]
       def config
-        parser_options.fetch(:config)
+        options.fetch(:config)
       end
 
       private

data/lib/click_house/middleware/summary_middleware.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Middleware
+    class SummaryMiddleware < ResponseBase
+      Faraday::Response.register_middleware self => self
+
+      KEY = :summary
+
+      # @param env [Faraday::Env]
+      # @return [Response::Summary]
+      def self.extract(env)
+        env.custom_members.fetch(KEY)
+      end
+
+      # @param env [Faraday::Env]
+      def on_complete(env)
+        env.custom_members[KEY] = Response::Summary.new(
+          config,
+          headers: env.response_headers,
+          body: env.body.is_a?(Hash) ? env.body : {}
+        )
+      end
+    end
+  end
+end

data/lib/click_house/middleware.rb

@@ -3,6 +3,7 @@
 module ClickHouse
   module Middleware
     autoload :ResponseBase, 'click_house/middleware/response_base'
+    autoload :SummaryMiddleware, 'click_house/middleware/summary_middleware'
     autoload :Logging, 'click_house/middleware/logging'
     autoload :ParseCsv, 'click_house/middleware/parse_csv'
     autoload :ParseJsonOj, 'click_house/middleware/parse_json_oj'

data/lib/click_house/response/factory.rb

@@ -3,21 +3,49 @@
 module ClickHouse
   module Response
     class Factory
-      # @return [String, ResultSet]
-      # @params env [Faraday::Response]
-      def self.[](faraday)
+      KEY_META = 'meta'
+      KEY_DATA = 'data'
+
+      # @return [ResultSet]
+      # @params faraday [Faraday::Response]
+      # @params config [Config]
+      def self.response(faraday, config)
         body = faraday.body
 
-        return body if !body.is_a?(Hash) || !(body.key?('meta') && body.key?('data'))
+        # wrap to be able to use connection#select_one, connection#select_value
+        # with other formats like binary
+        return raw(faraday, config) unless body.is_a?(Hash)
+        return raw(faraday, config) unless body.key?(config.key(KEY_META)) && body.key?(config.key(KEY_DATA))
 
         ResultSet.new(
-          meta: body.fetch('meta'),
-          data: body.fetch('data'),
-          totals: body['totals'],
-          statistics: body['statistics'],
-          rows_before_limit_at_least: body['rows_before_limit_at_least']
+          config: config,
+          meta: body.fetch(config.key(KEY_META)),
+          data: body.fetch(config.key(KEY_DATA)),
+          summary: Middleware::SummaryMiddleware.extract(faraday.env)
+        )
+      end
+
+      # @return [ResultSet]
+      # Rae ResultSet (without type casting)
+      def self.raw(faraday, config)
+        ResultSet.raw(
+          config: config,
+          data: Util.array(faraday.body),
+          summary: Middleware::SummaryMiddleware.extract(faraday.env)
         )
       end
+
+      # Result of execution
+      # @return [Response::Summary]
+      # @params faraday [Faraday::Response]
+      def self.exec(faraday)
+        Middleware::SummaryMiddleware.extract(faraday.env)
+      end
+
+      # @return [Response::Summary]
+      def self.empty_exec(config)
+        Summary.new(config)
+      end
     end
   end
 end

data/lib/click_house/response/result_set.rb

@@ -6,26 +6,69 @@ module ClickHouse
       extend Forwardable
       include Enumerable
 
-      PLACEHOLDER_D = '%d'
-      PLACEHOLDER_S = '%s'
+      KEY_META_NAME = 'name'
+      KEY_META_TYPE = 'type'
 
       def_delegators :to_a,
                      :inspect, :each, :fetch, :length, :count, :size,
                      :first, :last, :[], :to_h
 
-      attr_reader :meta, :data, :totals, :statistics, :rows_before_limit_at_least
+      def_delegators :summary,
+                     :statistics, :headers,
+                     :totals, :rows_before_limit_at_least
 
+      attr_reader :config, :meta, :data, :summary
+
+      class << self
+        # @param config [Config]
+        # @return [ResultSet]
+        def raw(config:, data:, summary:)
+          new(config: config, data: data, to_a: data, meta: [], summary: summary)
+        end
+      end
+
+      # @param config [Config]
       # @param meta [Array]
       # @param data [Array]
-      # @param totals [Array|Hash|NilClass] Support for 'GROUP BY WITH TOTALS' modifier
-      #   https://clickhouse.tech/docs/en/sql-reference/statements/select/group-by/#with-totals-modifier
-      #   Hash in JSON format and Array in JSONCompact
-      def initialize(meta:, data:, totals: nil, statistics: nil, rows_before_limit_at_least: nil)
+      # @param summary [Response::Summary]
+      def initialize(config:, meta:, data:, summary:, to_a: nil)
+        @config = config
         @meta = meta
         @data = data
-        @totals = totals
-        @rows_before_limit_at_least = rows_before_limit_at_least
-        @statistics = Hash(statistics)
+        @summary = summary
+        @to_a = to_a
+      end
+
+      # @return [Array, Hash]
+      # @param data [Array, Hash]
+      def serialize(data)
+        case data
+        when Hash
+          serialize_one(data)
+        when Array
+          data.map(&method(:serialize_one))
+        else
+          raise ArgumentError, "expect Hash or Array, got: #{data.class}"
+        end
+      end
+
+      # @return [Hash]
+      # @param row [Hash]
+      def serialize_one(row)
+        row.each_with_object({}) do |(key, value), object|
+          object[key] = serialize_column(key, value)
+        end
+      end
+
+      # @param name [String] column name
+      # @param value [Any]
+      def serialize_column(name, value)
+        stmt = types.fetch(name)
+        serialize_type(stmt, value)
+      rescue KeyError => e
+        raise SerializeError, "field <#{name}> does not exists in table schema: #{types}", e.backtrace
+      rescue StandardError => e
+        raise SerializeError, "failed to serialize <#{name}> with #{stmt}, #{e.class}, #{e.message}", e.backtrace
       end
 
       def to_a
@@ -39,8 +82,11 @@ module ClickHouse
       # @return [Hash<String, Ast::Statement>]
       def types
         @types ||= meta.each_with_object({}) do |row, object|
-          object[row.fetch('name')] = begin
-            current = Ast::Parser.new(row.fetch('type')).parse
+          column = row.fetch(config.key(KEY_META_NAME))
+          # make symbol keys, if config.symbolize_keys is true,
+          # to be able to cast and serialize properly
+          object[config.key(column)] = begin
+            current = Ast::Parser.new(row.fetch(config.key(KEY_META_TYPE))).parse
             assign_type(current)
             current
           end
@@ -96,6 +142,42 @@ module ClickHouse
           cast_type(stmt.arguments.fetch(ix), item)
         end
       end
+
+      # @param stmt [Ast::Statement]
+      def serialize_type(stmt, value)
+        return serialize_container(stmt, value) if stmt.caster.container?
+        return serialize_map(stmt, value) if stmt.caster.map?
+        return serialize_tuple(stmt, Array(value)) if stmt.caster.tuple?
+
+        stmt.caster.serialize(value, *stmt.arguments.map(&:value))
+      end
+
+      # @param stmt [Ast::Statement]
+      def serialize_container(stmt, value)
+        stmt.caster.cast_each(value) do |item|
+          # TODO: raise an error if multiple arguments
+          serialize_type(stmt.arguments.first, item)
+        end
+      end
+
+      # @return [Hash]
+      # @param stmt [Ast::Statement]
+      # @param hash [Hash]
+      def serialize_map(stmt, hash)
+        raise ArgumentError, "expect hash got #{hash.class}" unless hash.is_a?(Hash)
+
+        key_type, value_type = stmt.arguments
+        hash.each_with_object({}) do |(key, value), object|
+          object[serialize_type(key_type, key)] = serialize_type(value_type, value)
+        end
+      end
+
+      # @param stmt [Ast::Statement]
+      def serialize_tuple(stmt, value)
+        value.map.with_index do |item, ix|
+          serialize_type(stmt.arguments.fetch(ix), item)
+        end
+      end
     end
   end
 end

data/lib/click_house/response/summary.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Response
+    class Summary
+      SUMMARY_HEADER = 'x-clickhouse-summary'
+      KEY_TOTALS = 'totals'
+      KEY_STATISTICS = 'statistics'
+      KEY_ROWS_BEFORE_LIMIT_AT_LEAST = 'rows_before_limit_at_least'
+      KEY_STAT_ELAPSED = 'elapsed'
+
+      attr_reader :config,
+                  :headers,
+                  :summary,
+                  # {:elapsed=>0.387287e-3, :rows_read=>0, :bytes_read=>0}}
+                  :statistics,
+                  :totals,
+                  :rows_before_limit_at_least
+
+      # @param config [Config]
+      # @param headers [Faraday::Utils::Headers]
+      # @param body [Hash]
+      # TOTALS [Array|Hash|NilClass] Support for 'GROUP BY WITH TOTALS' modifier
+      #   https://clickhouse.tech/docs/en/sql-reference/statements/select/group-by/#with-totals-modifier
+      #   Hash in JSON format and Array in JSONCompact
+      def initialize(config, headers: Faraday::Utils::Headers.new, body: {})
+        @headers = headers
+        @config = config
+        @statistics = body.fetch(config.key(KEY_STATISTICS), {})
+        @totals = body[config.key(KEY_TOTALS)]
+        @rows_before_limit_at_least = body[config.key(KEY_ROWS_BEFORE_LIMIT_AT_LEAST)]
+        @summary = parse_summary(headers[SUMMARY_HEADER])
+      end
+
+      # @return [Integer]
+      def read_rows
+        summary[config.key('read_rows')].to_i
+      end
+
+      # @return [Integer]
+      def read_bytes
+        summary[config.key('read_bytes')].to_i
+      end
+
+      # @return [String]
+      def read_bytes_pretty
+        Util::Pretty.size(read_bytes)
+      end
+
+      # @return [Integer]
+      def written_rows
+        summary[config.key('written_rows')].to_i
+      end
+
+      # @return [Integer]
+      def written_bytes
+        summary[config.key('written_bytes')].to_i
+      end
+
+      # @return [String]
+      def written_bytes_pretty
+        Util::Pretty.size(written_bytes)
+      end
+
+      # @return [Integer]
+      def total_rows_to_read
+        summary[config.key('total_rows_to_read')].to_i
+      end
+
+      # @return [Integer]
+      def result_rows
+        summary[config.key('result_rows')].to_i
+      end
+
+      # @return [Integer]
+      def result_bytes
+        summary[config.key('result_bytes')].to_i
+      end
+
+      # @return [Float]
+      def elapsed
+        statistics[config.key(KEY_STAT_ELAPSED)].to_f
+      end
+
+      # @return [String]
+      def elapsed_pretty
+        Util::Pretty.measure(elapsed * 1000)
+      end
+
+      private
+
+      # @return [Hash]
+      # {
+      #   "read_rows" => "1",
+      #   "read_bytes" => "23",
+      #   "written_rows" => "1",
+      #   "written_bytes" => "23",
+      #   "total_rows_to_read" => "0",
+      #   "result_rows" => "1",
+      #   "result_bytes" => "23",
+      # }
+      def parse_summary(value)
+        return {} if value.nil? || value.empty?
+
+        JSON.parse(value)
+      end
+    end
+  end
+end

data/lib/click_house/response.rb

@@ -4,6 +4,6 @@ module ClickHouse
   module Response
     autoload :Factory, 'click_house/response/factory'
     autoload :ResultSet, 'click_house/response/result_set'
-    autoload :Tokenize, 'click_house/response/tokenize'
+    autoload :Summary, 'click_house/response/summary'
   end
 end

data/lib/click_house/serializer/base.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Serializer
+    class Base
+      attr_reader :config
+
+      # @param config [Config]
+      def initialize(config)
+        @config = config
+        on_setup
+      end
+
+      def dump(data)
+        raise NotImplementedError, __method__
+      end
+
+      # @return [String]
+      # @param data [Array]
+      def dump_each_row(data, sep = "\n")
+        data.map(&method(:dump)).join(sep)
+      end
+
+      private
+
+      # require external dependencies here
+      def on_setup
+        nil
+      end
+    end
+  end
+end

data/lib/click_house/serializer/json_oj_serializer.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Serializer
+    class JsonOjSerializer < Base
+      def dump(data)
+        Oj.dump(data, config.oj_dump_options)
+      end
+
+      private
+
+      def on_setup
+        require 'oj' unless defined?(Oj)
+      end
+    end
+  end
+end

data/lib/click_house/serializer/json_serializer.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Serializer
+    class JsonSerializer < Base
+      def dump(data)
+        JSON.dump(data)
+      end
+    end
+  end
+end

data/lib/click_house/serializer.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Serializer
+    autoload :Base, 'click_house/serializer/base'
+    autoload :JsonSerializer, 'click_house/serializer/json_serializer'
+    autoload :JsonOjSerializer, 'click_house/serializer/json_oj_serializer'
+  end
+end

data/lib/click_house/type/array_type.rb

@@ -7,6 +7,10 @@ module ClickHouse
         value.map(&block)
       end
 
+      def serialize_each(value, *_argv, &block)
+        value.map(&block)
+      end
+
       def container?
         true
       end

data/lib/click_house/type/base_type.rb

@@ -11,6 +11,10 @@ module ClickHouse
         raise NotImplementedError, __method__
       end
 
+      def serialize_each(_value, *)
+        raise NotImplementedError, __method__
+      end
+
       # @return [Boolean]
       # true if type contains another type like Nullable(T) or Array(T)
       def container?

data/lib/click_house/type/boolean_type.rb

@@ -7,7 +7,12 @@ module ClickHouse
       FALSE_VALUE = 0
 
       def cast(value)
-        value.to_i == TRUE_VALUE
+        case value
+        when TrueClass, FalseClass
+          value
+        else
+          value.to_i == TRUE_VALUE
+        end
       end
 
       def serialize(value)

data/lib/click_house/type/date_time64_type.rb

@@ -11,7 +11,7 @@ module ClickHouse
         end
       end
 
-      def serialize(value, precision = 3)
+      def serialize(value, precision = 3, _tz = nil)
         value.strftime("%Y-%m-%d %H:%M:%S.%#{precision}N")
       end
     end

data/lib/click_house/type/date_time_type.rb

@@ -11,7 +11,7 @@ module ClickHouse
         end
       end
 
-      def serialize(value)
+      def serialize(value, *)
         value.strftime('%Y-%m-%d %H:%M:%S')
       end
     end

data/lib/click_house/type/decimal_type.rb

@@ -3,14 +3,14 @@
 module ClickHouse
   module Type
     class DecimalType < BaseType
-      MAXIMUM = Float::DIG
+      MAXIMUM = Float::DIG.next
 
       # clickhouse:
       # P - precision. Valid range: [ 1 : 76 ]. Determines how many decimal digits number can have (including fraction).
       # S - scale. Valid range: [ 0 : P ]. Determines how many decimal digits fraction can have.
       #
       # when Oj parser @refs https://stackoverflow.com/questions/47885304/deserialise-json-numbers-as-bigdecimal
-      def cast(value, precision = Float::DIG, _scale = nil)
+      def cast(value, precision = MAXIMUM, _scale = nil)
         case value
         when BigDecimal
           value
@@ -21,8 +21,9 @@ module ClickHouse
         end
       end
 
-      def serialize(value, precision = Float::DIG, _scale = nil)
-        BigDecimal(value, precision.to_i).to_f unless value.nil?
+      # @return [BigDecimal]
+      def serialize(value, precision = MAXIMUM, _scale = nil)
+        cast(value, precision)
       end
     end
   end

data/lib/click_house/type/string_type.rb

@@ -7,7 +7,7 @@ module ClickHouse
         value.to_s unless value.nil?
       end
 
-      def serialize(value)
+      def serialize(value, *)
         value.to_s unless value.nil?
       end
     end

data/lib/click_house/util.rb

@@ -4,5 +4,12 @@ module ClickHouse
   module Util
     autoload :Statement, 'click_house/util/statement'
     autoload :Pretty, 'click_house/util/pretty'
+
+    module_function
+
+    # wraps
+    def array(input)
+      input.is_a?(Array) ? input : [input]
+    end
   end
 end

data/lib/click_house/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ClickHouse
-  VERSION = '2.0.2'
+  VERSION = '2.1.1'
 end

data/lib/click_house.rb

@@ -12,6 +12,7 @@ require 'active_support/core_ext/time/calculations'
 require 'click_house/version'
 require 'click_house/errors'
 require 'click_house/response'
+require 'click_house/serializer'
 require 'click_house/type'
 require 'click_house/middleware'
 require 'click_house/extend'
@@ -33,6 +34,10 @@ module ClickHouse
   add_type 'LowCardinality', Type::LowCardinalityType.new
   add_type 'Tuple', Type::TupleType.new
 
+  %w[Bool].each do |column|
+    add_type column, Type::BooleanType.new
+  end
+
   %w[Date].each do |column|
     add_type column, Type::DateType.new
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: click_house
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.1.1
 platform: ruby
 authors:
 - Aliaksandr Shylau
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-19 00:00:00.000000000 Z
+date: 2022-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
@@ -175,6 +175,7 @@ files:
 - lib/click_house/ast/parser.rb
 - lib/click_house/ast/statement.rb
 - lib/click_house/ast/ticker.rb
+- lib/click_house/benchmark/map_join.rb
 - lib/click_house/config.rb
 - lib/click_house/connection.rb
 - lib/click_house/definition.rb
@@ -199,9 +200,15 @@ files:
 - lib/click_house/middleware/parse_json_oj.rb
 - lib/click_house/middleware/raise_error.rb
 - lib/click_house/middleware/response_base.rb
+- lib/click_house/middleware/summary_middleware.rb
 - lib/click_house/response.rb
 - lib/click_house/response/factory.rb
 - lib/click_house/response/result_set.rb
+- lib/click_house/response/summary.rb
+- lib/click_house/serializer.rb
+- lib/click_house/serializer/base.rb
+- lib/click_house/serializer/json_oj_serializer.rb
+- lib/click_house/serializer/json_serializer.rb
 - lib/click_house/type.rb
 - lib/click_house/type/array_type.rb
 - lib/click_house/type/base_type.rb