click_house 1.7.0 → 2.0.0

data/lib/click_house/config.rb

@@ -16,7 +16,20 @@ module ClickHouse
       open_timeout: nil,
       ssl_verify: false,
       headers: {},
-      global_params: {}
+      global_params: {},
+      json_parser: ClickHouse::Middleware::ParseJson,
+      oj_load_options: {
+        mode: :custom,
+        allow_blank: true,
+        bigdecimal_as_decimal: false, # dump BigDecimal as a String
+        bigdecimal_load: :bigdecimal, # convert all decimal numbers to BigDecimal
+        empty_string: false,
+        second_precision: 6,
+        time_format: :ruby,
+      },
+      json_load_options: {
+        decimal_class: BigDecimal,
+      },
     }.freeze
 
     attr_accessor :adapter
@@ -33,6 +46,9 @@ module ClickHouse
     attr_accessor :ssl_verify
     attr_accessor :headers
     attr_accessor :global_params
+    attr_accessor :oj_load_options
+    attr_accessor :json_load_options
+    attr_accessor :json_parser # response middleware
 
     def initialize(params = {})
       assign(DEFAULTS.merge(params))

data/lib/click_house/connection.rb

@@ -46,6 +46,7 @@ module ClickHouse
       transport.post(compose('/', query.merge(database: database, **params)), body)
     end
 
+    # transport should work the same both with Faraday v1 and Faraday v2
     def transport
       @transport ||= Faraday.new(config.url!) do |conn|
         conn.options.timeout = config.timeout
@@ -53,10 +54,10 @@ module ClickHouse
         conn.headers = config.headers
         conn.ssl.verify = config.ssl_verify
         conn.request(:basic_auth, config.username, config.password) if config.auth?
-        conn.response :json, content_type: %r{application/json}
         conn.response Middleware::RaiseError
         conn.response Middleware::Logging, logger: config.logger!
-        conn.response Middleware::ParseCsv, content_type: %r{text/csv}
+        conn.response config.json_parser, content_type: %r{application/json}, parser_options: { config: config }
+        conn.response Middleware::ParseCsv, content_type: %r{text/csv}, parser_options: { config: config }
         conn.adapter config.adapter
       end
     end

data/lib/click_house/definition/column_set.rb

@@ -3,7 +3,9 @@
 module ClickHouse
   module Definition
     class ColumnSet
-      TYPES = ClickHouse.type_names(nullable: false).map { |s| s.sub('%s', "'%s'") }.freeze
+      TYPES = ClickHouse.types.each_with_object([]) do |(name, type), object|
+        object << name.sub('%s', "'%s'") if type.ddl?
+      end
 
       class << self
         # @input "DateTime('%s')"
@@ -15,7 +17,6 @@ module ClickHouse
 
       TYPES.each do |type|
         method_name = method_name_for_type(type)
-
         # t.Decimal :customer_id, nullable: true, default: ''
         # t.Decimal :money, 1, 2, nullable: true, default: ''
         class_eval <<-METHODS, __FILE__, __LINE__ + 1

data/lib/click_house/extend/configurable.rb

@@ -4,6 +4,8 @@ module ClickHouse
   module Extend
     module Configurable
       def config(&block)
+        yield(@config) if defined?(@config) && block
+
         @config ||= Config.new(&block)
       end
     end

data/lib/click_house/extend/type_definition.rb

@@ -3,22 +3,12 @@
 module ClickHouse
   module Extend
     module TypeDefinition
-      NULLABLE = 'Nullable'
-      NULLABLE_RE = /#{NULLABLE}/i.freeze
-      LOW_CARDINALITY = 'LowCardinality'
-
       def types
         @types ||= Hash.new(Type::UndefinedType.new)
       end
 
-      def add_type(type, klass, nullable: true)
+      def add_type(type, klass)
         types[type] = klass
-        types["#{NULLABLE}(#{type})"] = Type::NullableType.new(klass) if nullable
-      end
-
-      # @return [Enum<String>]
-      def type_names(nullable:)
-        nullable ? types.keys : types.keys.grep_v(NULLABLE_RE)
       end
     end
   end

data/lib/click_house/middleware/parse_csv.rb

@@ -2,15 +2,14 @@
 
 module ClickHouse
   module Middleware
-    class ParseCsv < FaradayMiddleware::ResponseMiddleware
+    class ParseCsv < ResponseBase
       Faraday::Response.register_middleware self => self
 
-      dependency do
-        require 'csv' unless defined?(CSV)
-      end
+      # @param env [Faraday::Env]
+      def on_complete(env)
+        return unless content_type?(env, content_type)
 
-      define_parser do |body, parser_options|
-        CSV.parse(body, **Hash.new(parser_options)) unless body.strip.empty?
+        env.body = env.body.strip.empty? ? nil : CSV.parse(env.body)
       end
     end
   end

data/lib/click_house/middleware/parse_json.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Middleware
+    class ParseJson < ResponseBase
+      Faraday::Response.register_middleware self => self
+
+      # @param env [Faraday::Env]
+      def on_complete(env)
+        return unless content_type?(env, content_type)
+
+        env.body = JSON.parse(env.body, config.json_load_options) unless env.body.strip.empty?
+      end
+    end
+  end
+end

data/lib/click_house/middleware/parse_json_oj.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Middleware
+    class ParseJsonOj < ResponseBase
+      Faraday::Response.register_middleware self => self
+
+      # @param env [Faraday::Env]
+      def on_complete(env)
+        return unless content_type?(env, content_type)
+
+        env.body = Oj.load(env.body, config.oj_load_options) unless env.body.strip.empty?
+      end
+
+      private
+
+      def on_setup
+        require 'oj' unless defined?(Oj)
+      end
+    end
+  end
+end

data/lib/click_house/middleware/raise_error.rb

@@ -3,28 +3,22 @@
 module ClickHouse
   module Middleware
     class RaiseError < Faraday::Middleware
-      SUCCEED_STATUSES = (200..299).freeze
       EXCEPTION_CODE_HEADER = 'x-clickhouse-exception-code'
 
       Faraday::Response.register_middleware self => self
 
-      def call(environment)
-        @app.call(environment).on_complete(&method(:on_complete))
+      # @param env [Faraday::Env]
+      def call(env)
+        super
       rescue Faraday::ConnectionFailed => e
         raise NetworkException, e.message, e.backtrace
       end
 
-      private
-
+      # @param env [Faraday::Env]
       def on_complete(env)
-        # Valid since Clickhouse 22.6
-        if env.response_headers.key?(EXCEPTION_CODE_HEADER)
-          raise DbException, env.body
+        if env.response_headers.include?(EXCEPTION_CODE_HEADER) || !env.success?
+          raise DbException, "[#{env.status}] #{env.body}"
         end
-
-        return if SUCCEED_STATUSES.include?(env.status)
-
-        raise DbException, "[#{env.status}] #{env.body}"
       end
     end
   end

data/lib/click_house/middleware/response_base.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Middleware
+    class ResponseBase < Faraday::Middleware
+      CONTENT_TYPE_HEADER = 'content-type'
+
+      attr_reader :parser_options
+      attr_reader :content_type
+
+      def initialize(app = nil, parser_options: {}, content_type: nil, preserve_raw: false)
+        super(app)
+        @parser_options = parser_options
+        @content_type = content_type
+        @preserve_raw = preserve_raw
+        on_setup
+      end
+
+      # @return [Boolean]
+      # @param env [Faraday::Env]
+      # @param regexp [NilClass, Regexp]
+      def content_type?(env, regexp)
+        case regexp
+        when NilClass
+          false
+        when Regexp
+          regexp.match?(String(env[:response_headers][CONTENT_TYPE_HEADER]))
+        else
+          raise ArgumentError, "expected regexp got #{regexp.class}"
+        end
+      end
+
+      # @return [Config]
+      def config
+        parser_options.fetch(:config)
+      end
+
+      private
+
+      def on_setup
+        # require external dependencies here
+      end
+    end
+  end
+end

data/lib/click_house/middleware.rb

@@ -2,8 +2,11 @@
 
 module ClickHouse
   module Middleware
+    autoload :ResponseBase, 'click_house/middleware/response_base'
     autoload :Logging, 'click_house/middleware/logging'
     autoload :ParseCsv, 'click_house/middleware/parse_csv'
+    autoload :ParseJsonOj, 'click_house/middleware/parse_json_oj'
+    autoload :ParseJson, 'click_house/middleware/parse_json'
     autoload :RaiseError, 'click_house/middleware/raise_error'
   end
 end

data/lib/click_house/response/result_set.rb

@@ -6,10 +6,6 @@ module ClickHouse
       extend Forwardable
       include Enumerable
 
-      TYPE_ARGV_DELIM = ','
-      NULLABLE = 'Nullable'
-      NULLABLE_TYPE_RE = /#{NULLABLE}\((.+)\)/i.freeze
-      ARG_D_RE = /\A-?\d+\Z/.freeze
       PLACEHOLDER_D = '%d'
       PLACEHOLDER_S = '%s'
 
@@ -19,38 +15,6 @@ module ClickHouse
 
       attr_reader :meta, :data, :totals, :statistics, :rows_before_limit_at_least
 
-      class << self
-        # @return [Array<String, Array>]
-        # * first element is name of "ClickHouse.types.keys"
-        # * second element is extra arguments that should to be passed to <cast> function
-        #
-        # @input "DateTime('Europe/Moscow')"
-        # @output "DateTime(%s)"
-        #
-        # @input "Nullable(Decimal(10, 5))"
-        # @output "Nullable(Decimal(%d, %d))"
-        #
-        # @input "Decimal(10, 5)"
-        # @output "Decimal(%d, %d)"
-        def extract_type_info(type)
-          type = type.gsub(NULLABLE_TYPE_RE, '\1')
-          nullable = Regexp.last_match(1)
-          argv = []
-
-          type = type.gsub(/\((.+)\)/, '')
-
-          if (match = Regexp.last_match(1))
-            argv = match.split("#{TYPE_ARGV_DELIM} ")
-            placeholder = argv.map do |value|
-              value.match?(ARG_D_RE) ? PLACEHOLDER_D : PLACEHOLDER_S
-            end
-            type = "#{type}(#{placeholder.join("#{TYPE_ARGV_DELIM} ")})"
-          end
-
-          [nullable ? "#{NULLABLE}(#{type})" : type, argv]
-        end
-      end
-
       # @param meta [Array]
       # @param data [Array]
       # @param totals [Array|Hash|NilClass] Support for 'GROUP BY WITH TOTALS' modifier
@@ -67,20 +31,69 @@ module ClickHouse
       def to_a
         @to_a ||= data.each do |row|
           row.each do |name, value|
-            casting = types.fetch(name)
-            row[name] = casting.fetch(:caster).cast(value, *casting.fetch(:arguments))
+            row[name] = cast_type(types.fetch(name), value)
           end
         end
       end
 
+      # @return [Hash<String, Ast::Statement>]
       def types
         @types ||= meta.each_with_object({}) do |row, object|
-          type_name, argv = self.class.extract_type_info(row.fetch('type'))
+          object[row.fetch('name')] = begin
+            current = Ast::Parser.new(row.fetch('type')).parse
+            assign_type(current)
+            current
+          end
+        end
+      end
+
+      private
+
+      # @param stmt [Ast::Statement]
+      def assign_type(stmt)
+        stmt.caster = ClickHouse.types[stmt.name]
+
+        if stmt.caster.is_a?(Type::UndefinedType)
+          placeholders = stmt.arguments.map(&:placeholder)
+          stmt.caster = ClickHouse.types["#{stmt.name}(#{placeholders.join(', ')})"]
+        end
+
+        stmt.arguments.each(&method(:assign_type))
+      end
+
+      # @param stmt [Ast::Statement]
+      def cast_type(stmt, value)
+        return cast_container(stmt, value) if stmt.caster.container?
+        return cast_map(stmt, Hash(value)) if stmt.caster.map?
+        return cast_tuple(stmt, Array(value)) if stmt.caster.tuple?
+
+        stmt.caster.cast(value, *stmt.arguments.map(&:value))
+      end
+
+      # @return [Hash]
+      # @param stmt [Ast::Statement]
+      # @param hash [Hash]
+      def cast_map(stmt, hash)
+        raise ArgumentError, "expect hash got #{hash.class}" unless hash.is_a?(Hash)
+
+        key_type, value_type = stmt.arguments
+        hash.each_with_object({}) do |(key, value), object|
+          object[cast_type(key_type, key)] = cast_type(value_type, value)
+        end
+      end
+
+      # @param stmt [Ast::Statement]
+      def cast_container(stmt, value)
+        stmt.caster.cast_each(value) do |item|
+          # TODO: raise an error if multiple arguments
+          cast_type(stmt.arguments.first, item)
+        end
+      end
 
-          object[row.fetch('name')] = {
-            caster: ClickHouse.types[type_name],
-            arguments: argv
-          }
+      # @param stmt [Ast::Statement]
+      def cast_tuple(stmt, value)
+        value.map.with_index do |item, ix|
+          cast_type(stmt.arguments.fetch(ix), item)
         end
       end
     end

data/lib/click_house/response.rb

@@ -4,5 +4,6 @@ module ClickHouse
   module Response
     autoload :Factory, 'click_house/response/factory'
     autoload :ResultSet, 'click_house/response/result_set'
+    autoload :Tokenize, 'click_house/response/tokenize'
   end
 end

data/lib/click_house/type/array_type.rb

@@ -3,22 +3,16 @@
 module ClickHouse
   module Type
     class ArrayType < BaseType
-      attr_reader :subtype
-
-      def initialize(subtype)
-        @subtype = subtype
+      def cast_each(value, *_argv, &block)
+        value.map(&block)
       end
 
-      def cast(value, *)
-        value
+      def container?
+        true
       end
 
-      def serialize(array, *argv)
-        return unless array.is_a?(Array)
-
-        array.map do |value|
-          subtype.serialize(value, *argv)
-        end
+      def ddl?
+        false
       end
     end
   end

data/lib/click_house/type/base_type.rb

@@ -7,9 +7,33 @@ module ClickHouse
         raise NotImplementedError, __method__
       end
 
-      def serialize(_value, *)
+      def cast_each(_value, *)
         raise NotImplementedError, __method__
       end
+
+      # @return [Boolean]
+      # true if type contains another type like Nullable(T) or Array(T)
+      def container?
+        false
+      end
+
+      # @return [Boolean]
+      # true if type is a Map
+      def map?
+        false
+      end
+
+      # @return [Boolean]
+      # true if type is a Tuple
+      def tuple?
+        false
+      end
+
+      # @return [Boolean]
+      # skip type from DDL statements
+      def ddl?
+        true
+      end
     end
   end
 end

data/lib/click_house/type/date_time64_type.rb

@@ -4,7 +4,11 @@ module ClickHouse
   module Type
     class DateTime64Type < BaseType
       def cast(value, _precision = nil, tz = nil)
-        DateTime.parse("#{value} #{tz}")
+        if tz
+          Time.find_zone(tz).parse(value)
+        else
+          Time.parse(value)
+        end
       end
 
       def serialize(value, precision = 3)

data/lib/click_house/type/date_time_type.rb

@@ -4,7 +4,11 @@ module ClickHouse
   module Type
     class DateTimeType < BaseType
       def cast(value, tz = nil)
-        DateTime.parse("#{value} #{tz}")
+        if tz
+          Time.find_zone(tz).parse(value)
+        else
+          Time.parse(value)
+        end
       end
 
       def serialize(value)

data/lib/click_house/type/decimal_type.rb

@@ -3,8 +3,22 @@
 module ClickHouse
   module Type
     class DecimalType < BaseType
+      MAXIMUM = Float::DIG
+
+      # clickhouse:
+      # P - precision. Valid range: [ 1 : 76 ]. Determines how many decimal digits number can have (including fraction).
+      # S - scale. Valid range: [ 0 : P ]. Determines how many decimal digits fraction can have.
+      #
+      # when Oj parser @refs https://stackoverflow.com/questions/47885304/deserialise-json-numbers-as-bigdecimal
       def cast(value, precision = Float::DIG, _scale = nil)
-        BigDecimal(value, precision.to_i)
+        case value
+        when BigDecimal
+          value
+        when String
+          BigDecimal(value)
+        else
+          BigDecimal(value, precision > MAXIMUM ? MAXIMUM : precision)
+        end
       end
 
       def serialize(value, precision = Float::DIG, _scale = nil)

data/lib/click_house/type/low_cardinality_type.rb

@@ -3,18 +3,12 @@
 module ClickHouse
   module Type
     class LowCardinalityType < BaseType
-      attr_reader :subtype
-
-      def initialize(subtype)
-        @subtype = subtype
-      end
-
-      def cast(*argv)
-        subtype.cast(*argv)
+      def cast_each(value, *_argv)
+        yield(value) unless value.nil?
       end
 
-      def serialize(*argv)
-        subtype.serialize(*argv)
+      def container?
+        true
       end
     end
   end

data/lib/click_house/type/map_type.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Type
+    class MapType < BaseType
+      def map?
+        true
+      end
+
+      def ddl?
+        false
+      end
+    end
+  end
+end

data/lib/click_house/type/nullable_type.rb

@@ -3,18 +3,16 @@
 module ClickHouse
   module Type
     class NullableType < BaseType
-      attr_reader :subtype
-
-      def initialize(subtype)
-        @subtype = subtype
+      def cast_each(value, *_argv)
+        yield(value) unless value.nil?
       end
 
-      def cast(*argv)
-        subtype.cast(*argv) unless argv.first.nil?
+      def container?
+        true
       end
 
-      def serialize(*argv)
-        subtype.serialize(*argv) unless argv.first.nil?
+      def ddl?
+        false
       end
     end
   end

data/lib/click_house/type/string_type.rb

@@ -3,7 +3,7 @@
 module ClickHouse
   module Type
     class StringType < BaseType
-      def cast(value)
+      def cast(value, *)
         value.to_s unless value.nil?
       end
 

data/lib/click_house/type/tuple_type.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Type
+    class TupleType < BaseType
+      def tuple?
+        true
+      end
+
+      def ddl?
+        false
+      end
+    end
+  end
+end

data/lib/click_house/type.rb

@@ -14,6 +14,8 @@ module ClickHouse
     autoload :DecimalType, 'click_house/type/decimal_type'
     autoload :FixedStringType, 'click_house/type/fixed_string_type'
     autoload :ArrayType, 'click_house/type/array_type'
+    autoload :TupleType, 'click_house/type/tuple_type'
+    autoload :MapType, 'click_house/type/map_type'
     autoload :StringType, 'click_house/type/string_type'
     autoload :IPType, 'click_house/type/ip_type'
     autoload :LowCardinalityType, 'click_house/type/low_cardinality_type'

data/lib/click_house/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ClickHouse
-  VERSION = '1.7.0'
+  VERSION = '2.0.0'
 end

data/lib/click_house.rb

@@ -8,13 +8,14 @@ require 'logger'
 require 'faraday'
 require 'forwardable'
 require 'bigdecimal'
-require 'faraday_middleware'
+require 'active_support/core_ext/time/calculations'
 require 'click_house/version'
 require 'click_house/errors'
 require 'click_house/response'
 require 'click_house/type'
 require 'click_house/middleware'
 require 'click_house/extend'
+require 'click_house/ast'
 require 'click_house/util'
 require 'click_house/definition'
 
@@ -26,6 +27,12 @@ module ClickHouse
   autoload :Config, 'click_house/config'
   autoload :Connection, 'click_house/connection'
 
+  add_type 'Array', Type::ArrayType.new
+  add_type 'Nullable', Type::NullableType.new
+  add_type 'Map', Type::MapType.new
+  add_type 'LowCardinality', Type::LowCardinalityType.new
+  add_type 'Tuple', Type::TupleType.new
+
   %w[Date].each do |column|
     add_type column, Type::DateType.new
   end
@@ -42,7 +49,7 @@ module ClickHouse
     add_type column, Type::DateTime64Type.new
   end
 
-  ['Decimal(%d, %d)', 'Decimal32(%d)', 'Decimal64(%d)', 'Decimal128(%d)'].each do |column|
+  ['Decimal(%d, %d)', 'Decimal32(%d)', 'Decimal64(%d)', 'Decimal128(%d)', 'Decimal256(%d)'].each do |column|
     add_type column, Type::DecimalType.new
   end