click_house 1.6.2 → 2.0.0

data/lib/click_house/ast/parser.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'stringio'
+
+module ClickHouse
+  module Ast
+    class Parser
+      OPEN = '('
+      CLOSED = ')'
+      COMMA = ','
+      SPACE = ' '
+
+      attr_reader :input
+
+      # @param input [String]
+      def initialize(input)
+        @input = input
+      end
+
+      # @refs https://clickhouse.com/docs/en/sql-reference/data-types/
+      # Map(String, Decimal(10, 5))
+      # Array(Array(Array(Array(Nullable(Int, String)))))
+      def parse
+        ticker = Ticker.new
+        control = false
+
+        input.each_char do |char|
+          # cases like (1,<space after comma> 3)
+          next if control && char == SPACE
+
+          case char
+          when OPEN
+            control = true
+            ticker.open
+          when CLOSED
+            control = true
+            ticker.close
+          when COMMA
+            control = true
+            ticker.comma
+          else
+            control = false
+            ticker.char(char)
+          end
+        end
+
+        # if a single type like "Int"
+        ticker.current.name! unless control
+        ticker.current
+      end
+    end
+  end
+end

data/lib/click_house/ast/statement.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require 'stringio'
+
+module ClickHouse
+  module Ast
+    class Statement
+      PLACEHOLDER_S = '%s'
+      PLACEHOLDER_D = '%d'
+      DIGIT_RE = /\A\d+\Z/.freeze
+
+      attr_reader :name
+      attr_accessor :caster
+
+      def initialize(name: '')
+        @buffer = ''
+        @name = name
+        @opened = false
+      end
+
+      # @param value [String]
+      def print(value)
+        @buffer = "#{@buffer}#{value}"
+      end
+
+      def name!
+        @name = @buffer
+        @buffer = ''
+      end
+
+      def argument!
+        add_argument(Statement.new(name: @buffer))
+        @buffer = ''
+      end
+
+      # @param st [Statement]
+      def add_argument(st)
+        arguments.push(st)
+      end
+
+      # @param other [Statement]
+      def merge(other)
+        if other.named?
+          add_argument(other)
+        else
+          @arguments = arguments.concat(other.arguments)
+        end
+      end
+
+      def named?
+        !@name.empty?
+      end
+
+      def buffer?
+        !@buffer.empty?
+      end
+
+      # @return [Array<Statement>]
+      def arguments
+        @arguments ||= []
+      end
+
+      def placeholder
+        return @placeholder if defined?(@placeholder)
+
+        @placeholder = digit? ? PLACEHOLDER_D : PLACEHOLDER_S
+      end
+
+      def digit?
+        name.match?(DIGIT_RE)
+      end
+
+      def value
+        @value ||=
+          case placeholder
+          when PLACEHOLDER_D
+            Integer(name)
+          when PLACEHOLDER_S
+            # remove leading and trailing quotes
+            name[1..-2]
+          else
+            raise "unknown value extractor for <#{placeholder}>"
+          end
+      end
+
+      def to_s
+        out = StringIO.new
+        out.print(name.empty? ? 'NO_NAME' : name)
+        out.print("<#{@buffer}>") unless @buffer.empty?
+
+        if arguments.any?
+          out.print("(#{arguments.join(',')})")
+        end
+
+        out.string
+      end
+    end
+  end
+end

data/lib/click_house/ast/ticker.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'stringio'
+
+module ClickHouse
+  module Ast
+    class Ticker
+      attr_reader :root, :current
+
+      def initialize
+        @current = Statement.new
+      end
+
+      def open
+        current.name!
+        opened.push(current)
+        @current = Statement.new
+      end
+
+      def comma
+        current.argument! if current.buffer?
+        opened.last.merge(current)
+        @current = Statement.new
+      end
+
+      def close
+        current.argument! unless current.named?
+        opened.last.merge(current)
+        @current = opened.pop
+      end
+
+      # @param char [String]
+      def char(char)
+        current.print(char)
+      end
+
+      def opened
+        @opened ||= []
+      end
+    end
+  end
+end

data/lib/click_house/ast.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Ast
+    autoload :Statement, 'click_house/ast/statement'
+    autoload :Ticker, 'click_house/ast/ticker'
+    autoload :Parser, 'click_house/ast/parser'
+  end
+end

data/lib/click_house/config.rb

@@ -16,7 +16,20 @@ module ClickHouse
       open_timeout: nil,
       ssl_verify: false,
       headers: {},
-      global_params: {}
+      global_params: {},
+      json_parser: ClickHouse::Middleware::ParseJson,
+      oj_load_options: {
+        mode: :custom,
+        allow_blank: true,
+        bigdecimal_as_decimal: false, # dump BigDecimal as a String
+        bigdecimal_load: :bigdecimal, # convert all decimal numbers to BigDecimal
+        empty_string: false,
+        second_precision: 6,
+        time_format: :ruby,
+      },
+      json_load_options: {
+        decimal_class: BigDecimal,
+      },
     }.freeze
 
     attr_accessor :adapter
@@ -33,6 +46,9 @@ module ClickHouse
     attr_accessor :ssl_verify
     attr_accessor :headers
     attr_accessor :global_params
+    attr_accessor :oj_load_options
+    attr_accessor :json_load_options
+    attr_accessor :json_parser # response middleware
 
     def initialize(params = {})
       assign(DEFAULTS.merge(params))

data/lib/click_house/connection.rb

@@ -46,6 +46,7 @@ module ClickHouse
       transport.post(compose('/', query.merge(database: database, **params)), body)
     end
 
+    # transport should work the same both with Faraday v1 and Faraday v2
     def transport
       @transport ||= Faraday.new(config.url!) do |conn|
         conn.options.timeout = config.timeout
@@ -55,8 +56,8 @@ module ClickHouse
         conn.request(:basic_auth, config.username, config.password) if config.auth?
         conn.response Middleware::RaiseError
         conn.response Middleware::Logging, logger: config.logger!
-        conn.response :json, content_type: %r{application/json}
-        conn.response Middleware::ParseCsv, content_type: %r{text/csv}
+        conn.response config.json_parser, content_type: %r{application/json}, parser_options: { config: config }
+        conn.response Middleware::ParseCsv, content_type: %r{text/csv}, parser_options: { config: config }
         conn.adapter config.adapter
       end
     end

data/lib/click_house/definition/column.rb

@@ -6,6 +6,7 @@ module ClickHouse
       attr_accessor :name
       attr_accessor :type
       attr_accessor :nullable
+      attr_accessor :low_cardinality
       attr_accessor :extensions
       attr_accessor :default
       attr_accessor :materialized
@@ -17,10 +18,12 @@ module ClickHouse
       end
 
       def to_s
-        nullable ? "#{name} Nullable(#{extension_type}) #{opts}" : "#{name} #{extension_type} #{opts}"
-      end
+        type = extension_type
+        type = "Nullable(#{type})" if nullable
+        type = "LowCardinality(#{type})" if low_cardinality
 
-      private
+        "#{name} #{type} #{opts}"
+      end
 
       def opts
         options = {

data/lib/click_house/definition/column_set.rb

@@ -3,7 +3,9 @@
 module ClickHouse
   module Definition
     class ColumnSet
-      TYPES = ClickHouse.type_names(nullable: false).map { |s| s.sub('%s', "'%s'") }.freeze
+      TYPES = ClickHouse.types.each_with_object([]) do |(name, type), object|
+        object << name.sub('%s', "'%s'") if type.ddl?
+      end
 
       class << self
         # @input "DateTime('%s')"
@@ -15,7 +17,6 @@ module ClickHouse
 
       TYPES.each do |type|
         method_name = method_name_for_type(type)
-
         # t.Decimal :customer_id, nullable: true, default: ''
         # t.Decimal :money, 1, 2, nullable: true, default: ''
         class_eval <<-METHODS, __FILE__, __LINE__ + 1

data/lib/click_house/extend/configurable.rb

@@ -4,6 +4,8 @@ module ClickHouse
   module Extend
     module Configurable
       def config(&block)
+        yield(@config) if defined?(@config) && block
+
         @config ||= Config.new(&block)
       end
     end

data/lib/click_house/extend/connection_inserting.rb

@@ -14,8 +14,8 @@ module ClickHouse
       # end
       #
       # == Example with a param
-      # subject.insert('rspec', values: [{ name: 'Sun', id: 1 }, { name: 'Moon', id: 2 }])
-      def insert(table, columns: [], values: [])
+      # subject.insert('rspec', values: [{ name: 'Sun', id: 1 }, { name: 'Moon', id: 2 }], format: 'JSONStringsEachRow')
+      def insert(table, columns: [], values: [], format: 'JSONEachRow')
         yield(values) if block_given?
 
         body = if columns.empty?
@@ -26,7 +26,7 @@ module ClickHouse
 
         return EMPTY_INSERT if values.empty?
 
-        execute("INSERT INTO #{table} FORMAT JSONEachRow", body.join("\n")).success?
+        execute("INSERT INTO #{table} FORMAT #{format}", body.join("\n")).success?
       end
     end
   end

data/lib/click_house/extend/type_definition.rb

@@ -3,21 +3,12 @@
 module ClickHouse
   module Extend
     module TypeDefinition
-      NULLABLE = 'Nullable'
-      NULLABLE_RE = /#{NULLABLE}/i.freeze
-
       def types
         @types ||= Hash.new(Type::UndefinedType.new)
       end
 
-      def add_type(type, klass, nullable: true)
+      def add_type(type, klass)
         types[type] = klass
-        types["#{NULLABLE}(#{type})"] = Type::NullableType.new(klass) if nullable
-      end
-
-      # @return [Enum<String>]
-      def type_names(nullable:)
-        nullable ? types.keys : types.keys.grep_v(NULLABLE_RE)
       end
     end
   end

data/lib/click_house/middleware/logging.rb

@@ -30,7 +30,7 @@ module ClickHouse
       def on_complete(env)
         summary = extract_summary(env.response_headers)
         logger.info("\e[1mSQL (#{duration_stats_log(env.body)})\e[0m #{query(env)};")
-        logger.debug(body) if body && !query_in_body?(env)
+        logger.debug(body) if body
         logger.info("\e[1mRead: #{summary.fetch(:read_rows)} rows, #{summary.fetch(:read_bytes)}. Written: #{summary.fetch(:written_rows)} rows, #{summary.fetch(:written_bytes)}\e[0m")
       end
       # rubocop:enable Layout/LineLength

data/lib/click_house/middleware/parse_csv.rb

@@ -2,15 +2,14 @@
 
 module ClickHouse
   module Middleware
-    class ParseCsv < FaradayMiddleware::ResponseMiddleware
+    class ParseCsv < ResponseBase
       Faraday::Response.register_middleware self => self
 
-      dependency do
-        require 'csv' unless defined?(CSV)
-      end
+      # @param env [Faraday::Env]
+      def on_complete(env)
+        return unless content_type?(env, content_type)
 
-      define_parser do |body, parser_options|
-        CSV.parse(body, **Hash.new(parser_options)) unless body.strip.empty?
+        env.body = env.body.strip.empty? ? nil : CSV.parse(env.body)
       end
     end
   end

data/lib/click_house/middleware/parse_json.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Middleware
+    class ParseJson < ResponseBase
+      Faraday::Response.register_middleware self => self
+
+      # @param env [Faraday::Env]
+      def on_complete(env)
+        return unless content_type?(env, content_type)
+
+        env.body = JSON.parse(env.body, config.json_load_options) unless env.body.strip.empty?
+      end
+    end
+  end
+end

data/lib/click_house/middleware/parse_json_oj.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Middleware
+    class ParseJsonOj < ResponseBase
+      Faraday::Response.register_middleware self => self
+
+      # @param env [Faraday::Env]
+      def on_complete(env)
+        return unless content_type?(env, content_type)
+
+        env.body = Oj.load(env.body, config.oj_load_options) unless env.body.strip.empty?
+      end
+
+      private
+
+      def on_setup
+        require 'oj' unless defined?(Oj)
+      end
+    end
+  end
+end

data/lib/click_house/middleware/raise_error.rb

@@ -3,22 +3,22 @@
 module ClickHouse
   module Middleware
     class RaiseError < Faraday::Middleware
-      SUCCEED_STATUSES = (200..299).freeze
+      EXCEPTION_CODE_HEADER = 'x-clickhouse-exception-code'
 
       Faraday::Response.register_middleware self => self
 
-      def call(environment)
-        @app.call(environment).on_complete(&method(:on_complete))
+      # @param env [Faraday::Env]
+      def call(env)
+        super
       rescue Faraday::ConnectionFailed => e
         raise NetworkException, e.message, e.backtrace
       end
 
-      private
-
+      # @param env [Faraday::Env]
       def on_complete(env)
-        return if SUCCEED_STATUSES.include?(env.status)
-
-        raise DbException, "[#{env.status}] #{env.body}"
+        if env.response_headers.include?(EXCEPTION_CODE_HEADER) || !env.success?
+          raise DbException, "[#{env.status}] #{env.body}"
+        end
       end
     end
   end

data/lib/click_house/middleware/response_base.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Middleware
+    class ResponseBase < Faraday::Middleware
+      CONTENT_TYPE_HEADER = 'content-type'
+
+      attr_reader :parser_options
+      attr_reader :content_type
+
+      def initialize(app = nil, parser_options: {}, content_type: nil, preserve_raw: false)
+        super(app)
+        @parser_options = parser_options
+        @content_type = content_type
+        @preserve_raw = preserve_raw
+        on_setup
+      end
+
+      # @return [Boolean]
+      # @param env [Faraday::Env]
+      # @param regexp [NilClass, Regexp]
+      def content_type?(env, regexp)
+        case regexp
+        when NilClass
+          false
+        when Regexp
+          regexp.match?(String(env[:response_headers][CONTENT_TYPE_HEADER]))
+        else
+          raise ArgumentError, "expected regexp got #{regexp.class}"
+        end
+      end
+
+      # @return [Config]
+      def config
+        parser_options.fetch(:config)
+      end
+
+      private
+
+      def on_setup
+        # require external dependencies here
+      end
+    end
+  end
+end

data/lib/click_house/middleware.rb

@@ -2,8 +2,11 @@
 
 module ClickHouse
   module Middleware
+    autoload :ResponseBase, 'click_house/middleware/response_base'
     autoload :Logging, 'click_house/middleware/logging'
     autoload :ParseCsv, 'click_house/middleware/parse_csv'
+    autoload :ParseJsonOj, 'click_house/middleware/parse_json_oj'
+    autoload :ParseJson, 'click_house/middleware/parse_json'
     autoload :RaiseError, 'click_house/middleware/raise_error'
   end
 end

data/lib/click_house/response/result_set.rb

@@ -6,9 +6,8 @@ module ClickHouse
       extend Forwardable
       include Enumerable
 
-      TYPE_ARGV_DELIM = ','
-      NULLABLE = 'Nullable'
-      NULLABLE_TYPE_RE = /#{NULLABLE}\((.+)\)/i.freeze
+      PLACEHOLDER_D = '%d'
+      PLACEHOLDER_S = '%s'
 
       def_delegators :to_a,
                      :inspect, :each, :fetch, :length, :count, :size,
@@ -16,36 +15,6 @@ module ClickHouse
 
       attr_reader :meta, :data, :totals, :statistics, :rows_before_limit_at_least
 
-      class << self
-        # @return [Array<String, Array>]
-        # * first element is name of "ClickHouse.types.keys"
-        # * second element is extra arguments that should to be passed to <cast> function
-        #
-        # @input "DateTime('Europe/Moscow')"
-        # @output "DateTime(%s)"
-        #
-        # @input "Nullable(Decimal(10, 5))"
-        # @output "Nullable(Decimal(%s, %s))"
-        #
-        # @input "Decimal(10, 5)"
-        # @output "Decimal(%s, %s)"
-        def extract_type_info(type)
-          type = type.gsub(NULLABLE_TYPE_RE, '\1')
-          nullable = Regexp.last_match(1)
-          argv = []
-
-          type = type.gsub(/\((.+)\)/, '')
-
-          if (match = Regexp.last_match(1))
-            counter = Array.new(match.count(TYPE_ARGV_DELIM).next) { '%s' }
-            type = "#{type}(#{counter.join("#{TYPE_ARGV_DELIM} ")})"
-            argv = match.split("#{TYPE_ARGV_DELIM} ")
-          end
-
-          [nullable ? "#{NULLABLE}(#{type})" : type, argv]
-        end
-      end
-
       # @param meta [Array]
       # @param data [Array]
       # @param totals [Array|Hash|NilClass] Support for 'GROUP BY WITH TOTALS' modifier
@@ -62,20 +31,69 @@ module ClickHouse
       def to_a
         @to_a ||= data.each do |row|
           row.each do |name, value|
-            casting = types.fetch(name)
-            row[name] = casting.fetch(:caster).cast(value, *casting.fetch(:arguments))
+            row[name] = cast_type(types.fetch(name), value)
           end
         end
       end
 
+      # @return [Hash<String, Ast::Statement>]
       def types
         @types ||= meta.each_with_object({}) do |row, object|
-          type_name, argv = self.class.extract_type_info(row.fetch('type'))
+          object[row.fetch('name')] = begin
+            current = Ast::Parser.new(row.fetch('type')).parse
+            assign_type(current)
+            current
+          end
+        end
+      end
+
+      private
+
+      # @param stmt [Ast::Statement]
+      def assign_type(stmt)
+        stmt.caster = ClickHouse.types[stmt.name]
+
+        if stmt.caster.is_a?(Type::UndefinedType)
+          placeholders = stmt.arguments.map(&:placeholder)
+          stmt.caster = ClickHouse.types["#{stmt.name}(#{placeholders.join(', ')})"]
+        end
+
+        stmt.arguments.each(&method(:assign_type))
+      end
+
+      # @param stmt [Ast::Statement]
+      def cast_type(stmt, value)
+        return cast_container(stmt, value) if stmt.caster.container?
+        return cast_map(stmt, Hash(value)) if stmt.caster.map?
+        return cast_tuple(stmt, Array(value)) if stmt.caster.tuple?
+
+        stmt.caster.cast(value, *stmt.arguments.map(&:value))
+      end
+
+      # @return [Hash]
+      # @param stmt [Ast::Statement]
+      # @param hash [Hash]
+      def cast_map(stmt, hash)
+        raise ArgumentError, "expect hash got #{hash.class}" unless hash.is_a?(Hash)
+
+        key_type, value_type = stmt.arguments
+        hash.each_with_object({}) do |(key, value), object|
+          object[cast_type(key_type, key)] = cast_type(value_type, value)
+        end
+      end
+
+      # @param stmt [Ast::Statement]
+      def cast_container(stmt, value)
+        stmt.caster.cast_each(value) do |item|
+          # TODO: raise an error if multiple arguments
+          cast_type(stmt.arguments.first, item)
+        end
+      end
 
-          object[row.fetch('name')] = {
-            caster: ClickHouse.types[type_name],
-            arguments: argv
-          }
+      # @param stmt [Ast::Statement]
+      def cast_tuple(stmt, value)
+        value.map.with_index do |item, ix|
+          cast_type(stmt.arguments.fetch(ix), item)
         end
       end
     end

data/lib/click_house/response.rb

@@ -4,5 +4,6 @@ module ClickHouse
   module Response
     autoload :Factory, 'click_house/response/factory'
     autoload :ResultSet, 'click_house/response/result_set'
+    autoload :Tokenize, 'click_house/response/tokenize'
   end
 end

data/lib/click_house/type/array_type.rb

@@ -3,22 +3,16 @@
 module ClickHouse
   module Type
     class ArrayType < BaseType
-      attr_reader :subtype
-
-      def initialize(subtype)
-        @subtype = subtype
+      def cast_each(value, *_argv, &block)
+        value.map(&block)
       end
 
-      def cast(value, *)
-        value
+      def container?
+        true
       end
 
-      def serialize(array, *argv)
-        return unless array.is_a?(Array)
-
-        array.map do |value|
-          subtype.serialize(value, *argv)
-        end
+      def ddl?
+        false
       end
     end
   end