click_house 1.0.0

data/lib/click_house/extend.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Extend
+    autoload :TypeDefinition, 'click_house/extend/type_definition'
+    autoload :Configurable, 'click_house/extend/configurable'
+    autoload :Connectible, 'click_house/extend/connectible'
+    autoload :ConnectionHealthy, 'click_house/extend/connection_healthy'
+    autoload :ConnectionDatabase, 'click_house/extend/connection_database'
+    autoload :ConnectionTable, 'click_house/extend/connection_table'
+    autoload :ConnectionSelective, 'click_house/extend/connection_selective'
+    autoload :ConnectionInserting, 'click_house/extend/connection_inserting'
+  end
+end

data/lib/click_house/extend/configurable.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Extend
+    module Configurable
+      def config(&block)
+        @config ||= Config.new(&block)
+      end
+    end
+  end
+end

data/lib/click_house/extend/connectible.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Extend
+    module Connectible
+      def connection=(connection)
+        @connection = connection
+      end
+
+      def connection
+        @connection ||= Connection.new(config.clone)
+      end
+    end
+  end
+end

data/lib/click_house/extend/connection_database.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Extend
+    module ConnectionDatabase
+      # @return [Array<String>]
+      def databases
+        Array(execute('SHOW DATABASES FORMAT CSV', database: nil).body).tap(&:flatten!)
+      end
+
+      def create_database(name, if_not_exists: false, cluster: nil, engine: nil)
+        sql = 'CREATE DATABASE %<exists>s %<name>s %<cluster>s %<engine>s'
+
+        pattern = {
+          name: name,
+          exists: Util::Statement.ensure(if_not_exists, 'IF NOT EXISTS'),
+          cluster: Util::Statement.ensure(cluster, "ON CLUSTER #{cluster}"),
+          engine: Util::Statement.ensure(engine, "ENGINE = #{engine}")
+        }
+
+        execute(format(sql, pattern), database: nil).success?
+      end
+
+      def drop_database(name, if_exists: false, cluster: nil)
+        sql = 'DROP DATABASE %<exists>s %<name>s %<cluster>s'
+
+        pattern = {
+          name: name,
+          exists: Util::Statement.ensure(if_exists, 'IF EXISTS'),
+          cluster: Util::Statement.ensure(cluster, "ON CLUSTER #{cluster}"),
+        }
+
+        execute(format(sql, pattern), database: nil).success?
+      end
+    end
+  end
+end

data/lib/click_house/extend/connection_healthy.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Extend
+    module ConnectionHealthy
+      def ping
+        # without +send_progress_in_http_headers: nil+ DB::Exception: Empty query returns
+        get(database: nil, query: { send_progress_in_http_headers: nil }).success?
+      end
+
+      def replicas_status
+        get('/replicas_status', database: nil).success?
+      end
+    end
+  end
+end

data/lib/click_house/extend/connection_inserting.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Extend
+    module ConnectionInserting
+      def insert(table, columns:, values: [])
+        yield(values) if block_given?
+        body = "#{columns.to_csv}#{values.map(&:to_csv).join('')}"
+        execute("INSERT INTO #{table} FORMAT CSVWithNames", body).success?
+      end
+    end
+  end
+end

data/lib/click_house/extend/connection_selective.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Extend
+    module ConnectionSelective
+      # @return [ResultSet]
+      def select_all(sql)
+        response = execute(Util::Statement.format(sql, 'JSON'))
+        Response::Factory[response]
+      end
+
+      def select_value(sql)
+        response = execute(Util::Statement.format(sql, 'JSON'))
+        Array(Response::Factory[response].first).dig(0, -1)
+      end
+
+      def select_one(sql)
+        response = execute(Util::Statement.format(sql, 'JSON'))
+        Response::Factory[response].first
+      end
+    end
+  end
+end

data/lib/click_house/extend/connection_table.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Extend
+    module ConnectionTable
+      # @return [Array<String>]
+      def tables
+        Array(execute('SHOW TABLES FORMAT CSV').body).tap(&:flatten!)
+      end
+
+      # @return [ResultSet]
+      def describe_table(name)
+        Response::Factory[execute("DESCRIBE TABLE #{name} FORMAT JSON")]
+      end
+
+      # @return [Boolean]
+      def table_exists?(name, temporary: false)
+        sql = 'EXISTS %<temporary>s TABLE  %<name>s FORMAT CSV'
+
+        pattern = {
+          name: name,
+          temporary: Util::Statement.ensure(temporary, 'TEMPORARY')
+        }
+
+        Type::BooleanType.new.cast(execute(format(sql, pattern)).body.dig(0, 0))
+      end
+
+      def drop_table(name, temporary: false, if_exists: false, cluster: nil)
+        sql = 'DROP %<temporary>s TABLE %<exists>s %<name>s %<cluster>s'
+
+        pattern = {
+          name: name,
+          temporary: Util::Statement.ensure(temporary, 'TEMPORARY'),
+          exists: Util::Statement.ensure(if_exists, 'IF EXISTS'),
+          cluster: Util::Statement.ensure(cluster, "ON CLUSTER #{cluster}"),
+        }
+
+        execute(format(sql, pattern)).success?
+      end
+
+      # rubocop:disable Metrics/ParameterLists
+      def create_table(
+        name,
+        if_not_exists: false, cluster: nil,
+        partition: nil, order: nil, primary_key: nil, sample: nil, ttl: nil, settings: nil,
+        engine:,
+        &block
+      )
+        sql = <<~SQL
+          CREATE TABLE %<exists>s %<name>s %<cluster>s %<definition>s %<engine>s
+            %<partition>s
+            %<order>s
+            %<primary_key>s
+            %<sample>s
+            %<ttl>s
+            %<settings>s
+        SQL
+        definition = ClickHouse::Definition::ColumnSet.new(&block)
+
+        pattern = {
+          name: name,
+          exists: Util::Statement.ensure(if_not_exists, 'IF NOT EXISTS'),
+          definition: definition.to_s,
+          cluster: Util::Statement.ensure(cluster, "ON CLUSTER #{cluster}"),
+          partition: Util::Statement.ensure(partition, "PARTITION BY #{partition}"),
+          order: Util::Statement.ensure(order, "ORDER BY #{order}"),
+          primary_key: Util::Statement.ensure(primary_key, "PRIMARY KEY #{primary_key}"),
+          sample: Util::Statement.ensure(sample, "SAMPLE BY  #{sample}"),
+          ttl: Util::Statement.ensure(ttl, "TTL #{ttl}"),
+          settings: Util::Statement.ensure(settings, "SETTINGS #{settings}"),
+          engine: Util::Statement.ensure(engine, "ENGINE = #{engine}"),
+        }
+
+        puts format(sql, pattern)
+
+        execute(format(sql, pattern)).success?
+      end
+      # rubocop:enable Metrics/ParameterLists
+    end
+  end
+end

data/lib/click_house/extend/type_definition.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Extend
+    module TypeDefinition
+      def types
+        @types ||= Hash.new(Type::UndefinedType.new)
+      end
+
+      def add_type(type, klass)
+        types[type] = klass
+      end
+    end
+  end
+end

data/lib/click_house/middleware.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Middleware
+    autoload :Logging, 'click_house/middleware/logging'
+    autoload :ParseCsv, 'click_house/middleware/parse_csv'
+    autoload :RaiseError, 'click_house/middleware/raise_error'
+  end
+end

data/lib/click_house/middleware/logging.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Middleware
+    class Logging < Faraday::Middleware
+      Faraday::Response.register_middleware self => self
+
+      SUMMARY_HEADER = 'x-clickhouse-summary'
+
+      attr_reader :logger, :starting, :body
+
+      def initialize(app = nil, logger:)
+        @logger = logger
+        super(app)
+      end
+
+      def call(environment)
+        @starting = timestamp
+        @body = environment.body if log_body?
+        @app.call(environment).on_complete(&method(:on_complete))
+      end
+
+      private
+
+      def log_body?
+        logger.level == Logger::DEBUG
+      end
+
+      # rubocop:disable Metrics/LineLength
+      def on_complete(env)
+        summary = extract_summary(env.response_headers)
+        elapsed = duration
+        query = CGI.parse(env.url.query.to_s).dig('query', 0) || '[NO QUERY]'
+
+        logger.info("\e[1mSQL (#{Util::Pretty.measure(elapsed)})\e[0m #{query};")
+        logger.debug(body) if body
+        logger.info("\e[1mRead: #{summary.fetch(:read_rows)} rows, #{summary.fetch(:read_bytes)}. Written: #{summary.fetch(:written_rows)}, rows #{summary.fetch(:written_bytes)}\e[0m")
+      end
+      # rubocop:enable Metrics/LineLength
+
+      def duration
+        timestamp - starting
+      end
+
+      def timestamp
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      def extract_summary(headers)
+        JSON.parse(headers.fetch('x-clickhouse-summary', '{}')).tap do |summary|
+          summary[:read_rows] = summary['read_rows']
+          summary[:read_bytes] = Util::Pretty.size(summary['read_bytes'].to_i)
+          summary[:written_rows] = summary['written_rows']
+          summary[:written_bytes] = Util::Pretty.size(summary['written_bytes'].to_i)
+        end
+      rescue JSON::ParserError
+        {}
+      end
+    end
+  end
+end

data/lib/click_house/middleware/parse_csv.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Middleware
+    class ParseCsv < FaradayMiddleware::ResponseMiddleware
+      Faraday::Response.register_middleware self => self
+
+      dependency do
+        require 'csv' unless defined?(CSV)
+      end
+
+      define_parser do |body, parser_options|
+        CSV.parse(body, parser_options || {}) unless body.strip.empty?
+      end
+    end
+  end
+end

data/lib/click_house/middleware/raise_error.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Middleware
+    class RaiseError < Faraday::Middleware
+      SUCCEED_STATUSES = (200..299).freeze
+
+      Faraday::Response.register_middleware self => self
+
+      def call(environment)
+        @app.call(environment).on_complete(&method(:on_complete))
+      rescue Faraday::ConnectionFailed => e
+        raise NetworkException, e.message, e.backtrace
+      end
+
+      private
+
+      def on_complete(env)
+        return if SUCCEED_STATUSES.include?(env.status)
+
+        raise DbException, "[#{env.status}] #{env.body}"
+      end
+    end
+  end
+end

data/lib/click_house/response.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Response
+    autoload :Factory, 'click_house/response/factory'
+    autoload :ResultSet, 'click_house/response/result_set'
+  end
+end

data/lib/click_house/response/factory.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Response
+    class Factory
+      # @return [String, ResultSet]
+      # @params env [Faraday::Response]
+      def self.[](faraday)
+        body = faraday.body
+
+        return body if !body.is_a?(Hash) || !(body.key?('meta') && body.key?('data'))
+
+        ResultSet.new(meta: body.fetch('meta'), data: body.fetch('data'), statistics: body['statistics'])
+      end
+    end
+  end
+end

data/lib/click_house/response/result_set.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Response
+    class ResultSet
+      extend Forwardable
+      include Enumerable
+
+      TYPE_ARGV_DELIM = ','
+      NULLABLE = 'Nullable'
+
+      def_delegators :to_a, :each, :fetch
+
+      attr_reader :meta, :data, :statistics
+
+      class << self
+        # @return [Array<String, Array>]
+        # * first element is name of "ClickHouse.types.keys"
+        # * second element is extra arguments that should to be passed to <cast> function
+        #
+        # @input "DateTime('Europe/Moscow')"
+        # @output "DateTime(%s)"
+        #
+        # @input "Nullable(Decimal(10, 5))"
+        # @output "Nullable(Decimal(%s, %s))"
+        #
+        # @input "Decimal(10, 5)"
+        # @output "Decimal(%s, %s)"
+        def extract_type_info(type)
+          type = type.gsub(/#{NULLABLE}\((.+)\)/i, '\1')
+          nullable = Regexp.last_match(1)
+          argv = []
+
+          type = type.gsub(/\((.+)\)/, '')
+
+          if (match = Regexp.last_match(1))
+            counter = Array.new(match.count(TYPE_ARGV_DELIM).next) { '%s' }
+            type = "#{type}(#{counter.join("#{TYPE_ARGV_DELIM} ")})"
+            argv = match.split("#{TYPE_ARGV_DELIM} ")
+          end
+
+          [nullable ? "#{NULLABLE}(#{type})" : type, argv]
+        end
+      end
+
+      # @param meta [Array]
+      # @param data [Array]
+      def initialize(meta:, data:, statistics: nil)
+        @meta = meta
+        @data = data
+        @statistics = Hash(statistics)
+      end
+
+      def to_a
+        @to_a ||= data.each do |row|
+          row.each do |name, value|
+            casting = types.fetch(name)
+            row[name] = casting.fetch(:caster).cast(value, *casting.fetch(:arguments))
+          end
+        end
+      end
+
+      def types
+        @types ||= meta.each_with_object({}) do |row, object|
+          type_name, argv = self.class.extract_type_info(row.fetch('type'))
+
+          object[row.fetch('name')] = {
+            caster: ClickHouse.types[type_name],
+            arguments: argv
+          }
+        end
+      end
+
+      def inspect
+        to_a
+      end
+    end
+  end
+end