click_house-client 0.2.3 → 0.3.0

data/lib/click_house/client/database.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Client
+    class Database
+      attr_reader :database
+
+      def initialize(database:, url:, username:, password:, variables: {})
+        @database = database
+        @url = url
+        @username = username
+        @password = password
+        @variables = {
+          database:,
+          enable_http_compression: 1 # enable HTTP compression by default
+        }.merge(variables).freeze
+      end
+
+      def uri
+        @uri ||= build_custom_uri
+      end
+
+      def build_custom_uri(extra_variables: {})
+        parsed = Addressable::URI.parse(@url)
+        parsed.query_values = @variables.merge(extra_variables)
+        parsed
+      end
+
+      def headers
+        @headers ||= {
+          'X-ClickHouse-User' => @username,
+          'X-ClickHouse-Key' => @password,
+          'X-ClickHouse-Format' => 'JSON' # always return JSON data
+        }.freeze
+      end
+
+      def with_default_database
+        self.class.new(
+          database: 'default',
+          url: @url,
+          username: @username,
+          password: @password,
+          variables: @variables.merge(database: 'default')
+        )
+      end
+    end
+  end
+end

data/lib/click_house/client/formatter.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Client
+    class Formatter
+      DEFAULT = ->(value) { value }
+
+      BASIC_TYPE_CASTERS = {
+        'UInt64' => ->(value) { Integer(value) },
+        "DateTime64(6, 'UTC')" => ->(value) { ActiveSupport::TimeZone['UTC'].parse(value) },
+        "IntervalSecond" => ->(value) { ActiveSupport::Duration.build(value.to_i) },
+        "IntervalMillisecond" => ->(value) { ActiveSupport::Duration.build(value.to_i / 1000.0) }
+      }.freeze
+
+      TYPE_CASTERS = BASIC_TYPE_CASTERS.merge(
+        BASIC_TYPE_CASTERS.transform_keys { |type| "Nullable(#{type})" }
+          .transform_values { |caster| ->(value) { value.nil? ? nil : caster.call(value) } }
+      )
+
+      def self.format(result)
+        name_type_mapping = result['meta'].each_with_object({}) do |column, hash|
+          hash[column['name']] = column['type']
+        end
+
+        result['data'].map do |row|
+          row.each_with_object({}) do |(column, value), casted_row|
+            caster = TYPE_CASTERS.fetch(name_type_mapping[column], DEFAULT)
+
+            casted_row[column] = caster.call(value)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/click_house/client/query.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Client
+    class Query < QueryLike
+      SUBQUERY_PLACEHOLDER_REGEX = /{\w+:Subquery}/ # example: {var:Subquery}, special "internal" type for subqueries
+      PLACEHOLDER_REGEX = /{\w+:(\w|[()])+}/ # example: {var:UInt8} or {var:Array(UInt8)}
+      PLACEHOLDER_NAME_REGEX = /{(\w+):/ # example: {var:UInt8} => var
+
+      def self.build(query)
+        return query if query.is_a?(ClickHouse::Client::QueryLike)
+
+        new(raw_query: query)
+      end
+
+      def initialize(raw_query:, placeholders: {})
+        raise QueryError, 'Empty query string given' if raw_query.blank?
+
+        @raw_query = raw_query
+        @placeholders = placeholders || {}
+      end
+
+      # List of placeholders to be sent to ClickHouse for replacement.
+      # If there are subqueries, merge their placeholders as well.
+      def prepared_placeholders
+        all_placeholders = placeholders.select { |_, v| !v.is_a?(QueryLike) }
+        all_placeholders.transform_values! { |v| prepared_placeholder_value(v) }
+
+        placeholders.each_value do |value|
+          next unless value.is_a?(QueryLike)
+
+          all_placeholders.merge!(value.prepared_placeholders) do |key, a, b|
+            raise QueryError, "mismatching values for the '#{key}' placeholder: #{a} vs #{b}"
+          end
+        end
+
+        all_placeholders
+      end
+
+      # Placeholder replacement is handled by ClickHouse, only subquery placeholders
+      # will be replaced.
+      def to_sql
+        raw_query.gsub(SUBQUERY_PLACEHOLDER_REGEX) do |placeholder_in_query|
+          value = placeholder_value(placeholder_in_query)
+
+          if value.is_a?(QueryLike)
+            value.to_sql
+          else
+            placeholder_in_query
+          end
+        end
+      end
+
+      def to_redacted_sql(bind_index_manager = BindIndexManager.new)
+        raw_query.gsub(PLACEHOLDER_REGEX) do |placeholder_in_query|
+          value = placeholder_value(placeholder_in_query)
+
+          if value.is_a?(QueryLike)
+            value.to_redacted_sql(bind_index_manager)
+          else
+            bind_index_manager.next_bind_str
+          end
+        end
+      end
+
+      private
+
+      attr_reader :raw_query, :placeholders
+
+      def placeholder_value(placeholder_in_query)
+        placeholder = placeholder_in_query[PLACEHOLDER_NAME_REGEX, 1]
+        placeholders.fetch(placeholder.to_sym)
+      end
+
+      def prepared_placeholder_value(value)
+        return value unless value.is_a?(Array)
+
+        Quoting.quote(value)
+      end
+    end
+  end
+end

data/lib/click_house/client/query_builder.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require 'active_record'
+
+module ClickHouse
+  module Client
+    class QueryBuilder < QueryLike
+      attr_reader :table
+      attr_accessor :conditions, :manager
+
+      VALID_NODES = [
+        Arel::Nodes::In,
+        Arel::Nodes::Equality,
+        Arel::Nodes::LessThan,
+        Arel::Nodes::LessThanOrEqual,
+        Arel::Nodes::GreaterThan,
+        Arel::Nodes::GreaterThanOrEqual,
+        Arel::Nodes::NamedFunction
+      ].freeze
+
+      def initialize(table_name)
+        @table = Arel::Table.new(table_name)
+        @manager = Arel::SelectManager.new(Arel::Table.engine).from(@table).project(Arel.star)
+        @conditions = []
+      end
+
+      # The `where` method currently only supports IN and equal to queries along
+      # with above listed VALID_NODES.
+      # For example, using a range (start_date..end_date) will result in incorrect SQL.
+      # If you need to query a range, use greater than and less than conditions with Arel.
+      #
+      # Correct usage:
+      #   query.where(query.table[:created_at].lteq(Date.today)).to_sql
+      #   "SELECT * FROM \"table\" WHERE \"table\".\"created_at\" <= '2023-08-01'"
+      #
+      # This also supports array conditions which will result in an IN query.
+      #   query.where(entity_id: [1,2,3]).to_sql
+      #   "SELECT * FROM \"table\" WHERE \"table\".\"entity_id\" IN (1, 2, 3)"
+      #
+      # Range support and more `Arel::Nodes` could be considered for future iterations.
+      # @return [ClickHouse::QueryBuilder] New instance of query builder.
+      def where(conditions)
+        validate_condition_type!(conditions)
+
+        deep_clone.tap do |new_instance|
+          if conditions.is_a?(Arel::Nodes::Node)
+            new_instance.conditions << conditions
+          else
+            add_conditions_to(new_instance, conditions)
+          end
+        end
+      end
+
+      def select(*fields)
+        deep_clone.tap do |new_instance|
+          existing_fields = new_instance.manager.projections.filter_map do |projection|
+            if projection.is_a?(Arel::Attributes::Attribute)
+              projection.name.to_s
+            elsif projection.to_s == '*'
+              nil
+            end
+          end
+
+          new_projections = (existing_fields + fields).map do |field|
+            if field.is_a?(Symbol)
+              field.to_s
+            else
+              field
+            end
+          end
+
+          new_instance.manager.projections = new_projections.uniq.map do |field|
+            if field.is_a?(Arel::Expressions)
+              field
+            else
+              new_instance.table[field.to_s]
+            end
+          end
+        end
+      end
+
+      def order(field, direction = :asc)
+        validate_order_direction!(direction)
+
+        deep_clone.tap do |new_instance|
+          table_field = new_instance.table[field]
+          new_order = direction.to_s.casecmp('desc').zero? ? table_field.desc : table_field.asc
+          new_instance.manager.order(new_order)
+        end
+      end
+
+      def group(*columns)
+        deep_clone.tap do |new_instance|
+          new_instance.manager.group(*columns)
+        end
+      end
+
+      def limit(count)
+        manager.take(count)
+        self
+      end
+
+      def offset(count)
+        manager.skip(count)
+        self
+      end
+
+      def to_sql
+        apply_conditions!
+
+        visitor = Arel::Visitors::ToSql.new(ClickHouse::Client::ArelEngine.new)
+        visitor.accept(manager.ast, Arel::Collectors::SQLString.new).value
+      end
+
+      def to_redacted_sql(bind_index_manager = ClickHouse::Client::BindIndexManager.new)
+        ClickHouse::Client::Redactor.redact(self, bind_index_manager)
+      end
+
+      private
+
+      def validate_condition_type!(condition)
+        return unless condition.is_a?(Arel::Nodes::Node) && VALID_NODES.exclude?(condition.class)
+
+        raise ArgumentError, "Unsupported Arel node type for QueryBuilder: #{condition.class.name}"
+      end
+
+      def add_conditions_to(instance, conditions)
+        conditions.each do |key, value|
+          instance.conditions << if value.is_a?(Array)
+                                   instance.table[key].in(value)
+                                 else
+                                   instance.table[key].eq(value)
+                                 end
+        end
+      end
+
+      def deep_clone
+        self.class.new(table.name).tap do |new_instance|
+          new_instance.manager = manager.clone
+          new_instance.conditions = conditions.map(&:clone)
+        end
+      end
+
+      def apply_conditions!
+        manager.constraints.clear
+        conditions.each { |condition| manager.where(condition) }
+      end
+
+      def validate_order_direction!(direction)
+        return if %w[asc desc].include?(direction.to_s.downcase)
+
+        raise ArgumentError, "Invalid order direction '#{direction}'. Must be :asc or :desc"
+      end
+    end
+  end
+end

data/lib/click_house/client/query_like.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Client
+    class QueryLike
+      # Build a SQL string that can be executed on a ClickHouse database.
+      def to_sql
+        raise NotImplementedError
+      end
+
+      # Redacted version of the SQL query generated by the to_sql method where the
+      # placeholders are stripped. These queries are meant to be exported to external
+      # log aggregation systems.
+      def to_redacted_sql(bind_index_manager = BindIndexManager.new)
+        raise NotImplementedError
+      end
+
+      # Override when placeholders should be supported
+      def prepared_placeholders
+        {}
+      end
+
+      private
+
+      # Override when placeholders should be supported
+      def placeholders
+        {}
+      end
+    end
+  end
+end

data/lib/click_house/client/quoting.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Client
+    module Quoting
+      class << self
+        def quote(value)
+          case value
+          when Numeric then value.to_s
+          when String, Symbol then "'#{value.gsub('\\', '\&\&').gsub("'", "''")}'"
+          when Array then "[#{value.map { |v| quote(v) }.join(',')}]"
+          when nil then "NULL"
+          else quote_str(value.to_s)
+          end
+        end
+
+        private
+
+        def quote_str(value)
+          "'#{value.gsub("'", "''")}'"
+        end
+      end
+    end
+  end
+end

data/lib/click_house/client/redactor.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Client
+    module Redactor
+      # Redacts the SQL query represented by the query builder.
+      #
+      # @param query_builder [::ClickHouse::Querybuilder] The query builder object to be redacted.
+      # @return [String] The redacted SQL query as a string.
+      # @raise [ArgumentError] when the condition in the query is of an unsupported type.
+      #
+      # Example:
+      #   query_builder = ClickHouse::QueryBuilder.new('users').where(name: 'John Doe')
+      #   redacted_query = ClickHouse::Redactor.redact(query_builder)
+      #   # The redacted_query will contain the SQL query with values replaced by placeholders.
+      #   output: "SELECT * FROM \"users\" WHERE \"users\".\"name\" = $1"
+      def self.redact(query_builder, bind_manager = ClickHouse::Client::BindIndexManager.new)
+        cloned_query_builder = query_builder.clone
+
+        cloned_query_builder.conditions = cloned_query_builder.conditions.map do |condition|
+          redact_condition(condition, bind_manager)
+        end
+
+        cloned_query_builder.manager.constraints.clear
+        cloned_query_builder.conditions.each do |condition|
+          cloned_query_builder.manager.where(condition)
+        end
+
+        visitor = Arel::Visitors::ToSql.new(ClickHouse::Client::ArelEngine.new)
+        visitor.accept(cloned_query_builder.manager.ast, Arel::Collectors::SQLString.new).value
+      end
+
+      def self.redact_condition(condition, bind_manager)
+        case condition
+        when Arel::Nodes::In
+          condition.left.in(Array.new(condition.right.size) { Arel.sql(bind_manager.next_bind_str) })
+        when Arel::Nodes::Equality
+          condition.left.eq(Arel.sql(bind_manager.next_bind_str))
+        when Arel::Nodes::LessThan
+          condition.left.lt(Arel.sql(bind_manager.next_bind_str))
+        when Arel::Nodes::LessThanOrEqual
+          condition.left.lteq(Arel.sql(bind_manager.next_bind_str))
+        when Arel::Nodes::GreaterThan
+          condition.left.gt(Arel.sql(bind_manager.next_bind_str))
+        when Arel::Nodes::GreaterThanOrEqual
+          condition.left.gteq(Arel.sql(bind_manager.next_bind_str))
+        when Arel::Nodes::NamedFunction
+          redact_named_function(condition, bind_manager)
+        else
+          raise ArgumentError, "Unsupported Arel node type for Redactor: #{condition.class}"
+        end
+      end
+
+      def self.redact_named_function(condition, bind_manager)
+        redacted_condition =
+          Arel::Nodes::NamedFunction.new(condition.name, condition.expressions.dup)
+
+        case redacted_condition.name
+        when 'startsWith'
+          redacted_condition.expressions[1] = Arel.sql(bind_manager.next_bind_str)
+        else
+          redacted_condition.expressions = redacted_condition.expressions.map { Arel.sql(bind_manager.next_bind_str) }
+        end
+
+        redacted_condition
+      end
+    end
+  end
+end

data/lib/click_house/client/response.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Client
+    class Response
+      attr_reader :body, :headers
+
+      def initialize(body, http_status_code, headers = {})
+        @body = body
+        @http_status_code = http_status_code
+        @headers = headers
+      end
+
+      def success?
+        @http_status_code == 200
+      end
+    end
+  end
+end

data/lib/click_house/client/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Client
+    VERSION = "0.3.0"
+  end
+end

data/lib/click_house/client.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require 'addressable'
+require 'json'
+require 'active_support/deprecation'
+require 'active_support/time'
+require 'active_support/notifications'
+require 'active_support/core_ext/string'
+require 'active_support/core_ext/object/blank'
+require_relative "client/version"
+require_relative "client/database"
+require_relative "client/configuration"
+require_relative "client/bind_index_manager"
+require_relative "client/quoting"
+require_relative "client/arel_engine"
+require_relative "client/query_like"
+require_relative "client/query"
+require_relative "client/redactor"
+require_relative "client/query_builder"
+require_relative "client/formatter"
+require_relative "client/response"
+
+module ClickHouse
+  module Client
+    class << self
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      def configure
+        yield(configuration)
+        configuration.validate!
+      end
+    end
+
+    Error = Class.new(StandardError)
+    ConfigurationError = Class.new(Error)
+    DatabaseError = Class.new(Error)
+    QueryError = Class.new(Error)
+
+    def self.database_configured?(database, configuration = self.configuration)
+      !!configuration.databases[database]
+    end
+
+    # Executes a SELECT database query
+    def self.select(query, database, configuration = self.configuration)
+      instrumented_execute(query, database, configuration) do |response, instrument|
+        parsed_response = configuration.json_parser.parse(response.body)
+
+        instrument[:statistics] = parsed_response['statistics']&.symbolize_keys
+
+        Formatter.format(parsed_response)
+      end
+    end
+
+    # Executes any kinds of database query without returning any data (INSERT, DELETE)
+    def self.execute(query, database, configuration = self.configuration)
+      instrumented_execute(query, database, configuration) do |response, instrument|
+        expose_summary(response.headers, instrument)
+      end
+
+      true
+    end
+
+    # Inserts a gzip-compressed CSV to ClickHouse
+    #
+    # Usage:
+    #
+    # Create a compressed CSV file:
+    # > File.binwrite("my_csv.csv", ActiveSupport::Gzip.compress("id\n10\n20"))
+    #
+    # Invoke the INSERT query:
+    # > ClickHouse::Client.insert_csv('INSERT INTO events (id) FORMAT CSV', File.open("my_csv.csv"), :main)
+    def self.insert_csv(query, io, database, configuration = self.configuration)
+      db = lookup_database(configuration, database)
+
+      headers = db.headers.merge(
+        'Transfer-Encoding' => 'chunked',
+        'Content-Length' => File.size(io).to_s,
+        'Content-Encoding' => 'gzip'
+      )
+
+      query = ClickHouse::Client::Query.build(query)
+      ActiveSupport::Notifications.instrument('sql.click_house', { query:, database: }) do |instrument|
+        response = configuration.http_post_proc.call(
+          db.build_custom_uri(extra_variables: { query: query.to_sql }).to_s,
+          headers,
+          io
+        )
+        raise DatabaseError, response.body unless response.success?
+
+        expose_summary(response.headers, instrument)
+      end
+
+      true
+    end
+
+    private_class_method def self.expose_summary(headers, instrument)
+      return unless headers['x-clickhouse-summary']
+
+      instrument[:statistics] =
+        configuration.json_parser.parse(headers['x-clickhouse-summary']).symbolize_keys
+    end
+
+    private_class_method def self.lookup_database(configuration, database)
+      configuration.databases[database].tap do |db|
+        raise ConfigurationError, "The database '#{database}' is not configured" unless db
+      end
+    end
+
+    private_class_method def self.instrumented_execute(query, database, configuration)
+      db = lookup_database(configuration, database)
+
+      query = ClickHouse::Client::Query.build(query)
+
+      log_contents = configuration.log_proc.call(query)
+      configuration.logger.info(log_contents)
+
+      ActiveSupport::Notifications.instrument('sql.click_house', { query:, database: }) do |instrument|
+        # Use a multipart POST request where the placeholders are sent with the param_ prefix
+        # See: https://github.com/ClickHouse/ClickHouse/issues/8842
+        query_with_params = query.prepared_placeholders.transform_keys { |key| "param_#{key}" }
+        query_with_params['query'] = query.to_sql
+
+        response = configuration.http_post_proc.call(
+          db.uri.to_s,
+          db.headers,
+          query_with_params
+        )
+
+        raise DatabaseError, response.body unless response.success?
+
+        yield response, instrument
+      end
+    end
+  end
+end