chc-r8 0.3.2

data/lib/click_house/client/configuration.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Client
+    class Configuration
+      # Configuration options:
+      #
+      # *register_database* (method): registers a database, the following arguments are required:
+      #   - database: database name
+      #   - url: URL and port to the HTTP interface
+      #   - username
+      #   - password
+      #   - variables (optional): configuration for the client
+      #
+      # *http_post_proc*: A callable object for invoking the HTTP request.
+      #   The object must handle the following parameters: url, headers, body
+      #   and return a ClickHouse::Client::Response object.
+      #
+      # *json_parser*: object for parsing JSON strings, it should respond to the "parse" method
+      #
+      # *logger*: object for receiving logger commands. Default `$stdout`
+      # *log_proc*: any output (e.g. structure) to wrap around the query for every statement
+      #
+      # Example:
+      #
+      # ClickHouse::Client.configure do |c|
+      #   c.register_database(:main,
+      #     database: 'gitlab_clickhouse_test',
+      #     url: 'http://localhost:8123',
+      #     username: 'default',
+      #     password: 'clickhouse',
+      #     variables: {
+      #       join_use_nulls: 1 # treat JOINs as per SQL standard
+      #     }
+      #   )
+      #
+      #   c.logger = MyLogger.new
+      #   c.log_proc = ->(query) do
+      #     { query_body: query.to_redacted_sql }
+      #   end
+      #
+      #   c.http_post_proc = lambda do |url, headers, body|
+      #     options = {
+      #       headers: headers,
+      #       body: body,
+      #       allow_local_requests: false
+      #     }
+      #
+      #     response = Gitlab::HTTP.post(url, options)
+      #     ClickHouse::Client::Response.new(response.body, response.code)
+      #   end
+      #
+      #   c.json_parser = JSON
+      # end
+      attr_accessor :http_post_proc, :json_parser, :logger, :log_proc
+      attr_reader :databases
+
+      def initialize
+        @databases = {}
+        @http_post_proc = nil
+        @json_parser = JSON
+        @logger = ::Logger.new($stdout)
+        @log_proc = ->(query) { query.to_sql }
+      end
+
+      def register_database(name, **args)
+        raise ConfigurationError, "The database '#{name}' is already registered" if @databases.key?(name)
+
+        @databases[name] = Database.new(**args)
+      end
+
+      def validate!
+        raise ConfigurationError, "The 'http_post_proc' option is not configured" unless @http_post_proc
+        raise ConfigurationError, "The 'json_parser' option is not configured" unless @json_parser
+      end
+    end
+  end
+end

data/lib/click_house/client/database.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Client
+    class Database
+      attr_reader :database
+
+      def initialize(database:, url:, username:, password:, variables: {})
+        @database = database
+        @url = url
+        @username = username
+        @password = password
+        @variables = {
+          database:,
+          enable_http_compression: 1 # enable HTTP compression by default
+        }.merge(variables).freeze
+      end
+
+      def uri
+        @uri ||= build_custom_uri
+      end
+
+      def build_custom_uri(extra_variables: {})
+        parsed = Addressable::URI.parse(@url)
+        parsed.query_values = @variables.merge(extra_variables)
+        parsed
+      end
+
+      def headers
+        @headers ||= {
+          'X-ClickHouse-User' => @username,
+          'X-ClickHouse-Key' => @password,
+          'X-ClickHouse-Format' => 'JSON' # always return JSON data
+        }.freeze
+      end
+
+      def with_default_database
+        self.class.new(
+          database: 'default',
+          url: @url,
+          username: @username,
+          password: @password,
+          variables: @variables.merge(database: 'default')
+        )
+      end
+    end
+  end
+end

data/lib/click_house/client/formatter.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Client
+    class Formatter
+      DEFAULT = ->(value) { value }
+
+      BASIC_TYPE_CASTERS = {
+        'UInt64' => ->(value) { Integer(value) },
+        "DateTime64(6, 'UTC')" => ->(value) { ActiveSupport::TimeZone['UTC'].parse(value) },
+        "IntervalSecond" => ->(value) { ActiveSupport::Duration.build(value.to_i) },
+        "IntervalMillisecond" => ->(value) { ActiveSupport::Duration.build(value.to_i / 1000.0) }
+      }.freeze
+
+      TYPE_CASTERS = BASIC_TYPE_CASTERS.merge(
+        BASIC_TYPE_CASTERS.transform_keys { |type| "Nullable(#{type})" }
+          .transform_values { |caster| ->(value) { value.nil? ? nil : caster.call(value) } }
+      )
+
+      def self.format(result)
+        name_type_mapping = result['meta'].each_with_object({}) do |column, hash|
+          hash[column['name']] = column['type']
+        end
+
+        result['data'].map do |row|
+          row.each_with_object({}) do |(column, value), casted_row|
+            caster = TYPE_CASTERS.fetch(name_type_mapping[column], DEFAULT)
+
+            casted_row[column] = caster.call(value)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/click_house/client/query.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Client
+    class Query < QueryLike
+      SUBQUERY_PLACEHOLDER_REGEX = /{\w+:Subquery}/ # example: {var:Subquery}, special "internal" type for subqueries
+      PLACEHOLDER_REGEX = /{\w+:(\w|[()])+}/ # example: {var:UInt8} or {var:Array(UInt8)}
+      PLACEHOLDER_NAME_REGEX = /{(\w+):/ # example: {var:UInt8} => var
+
+      def self.build(query)
+        return query if query.is_a?(ClickHouse::Client::QueryLike)
+
+        new(raw_query: query)
+      end
+
+      def initialize(raw_query:, placeholders: {})
+        raise QueryError, 'Empty query string given' if raw_query.blank?
+
+        @raw_query = raw_query
+        @placeholders = placeholders || {}
+      end
+
+      # List of placeholders to be sent to ClickHouse for replacement.
+      # If there are subqueries, merge their placeholders as well.
+      def prepared_placeholders
+        all_placeholders = placeholders.select { |_, v| !v.is_a?(QueryLike) }
+        all_placeholders.transform_values! { |v| prepared_placeholder_value(v) }
+
+        placeholders.each_value do |value|
+          next unless value.is_a?(QueryLike)
+
+          all_placeholders.merge!(value.prepared_placeholders) do |key, a, b|
+            raise QueryError, "mismatching values for the '#{key}' placeholder: #{a} vs #{b}"
+          end
+        end
+
+        all_placeholders
+      end
+
+      # Placeholder replacement is handled by ClickHouse, only subquery placeholders
+      # will be replaced.
+      def to_sql
+        raw_query.gsub(SUBQUERY_PLACEHOLDER_REGEX) do |placeholder_in_query|
+          value = placeholder_value(placeholder_in_query)
+
+          if value.is_a?(QueryLike)
+            value.to_sql
+          else
+            placeholder_in_query
+          end
+        end
+      end
+
+      def to_redacted_sql(bind_index_manager = BindIndexManager.new)
+        raw_query.gsub(PLACEHOLDER_REGEX) do |placeholder_in_query|
+          value = placeholder_value(placeholder_in_query)
+
+          if value.is_a?(QueryLike)
+            value.to_redacted_sql(bind_index_manager)
+          else
+            bind_index_manager.next_bind_str
+          end
+        end
+      end
+
+      private
+
+      attr_reader :raw_query, :placeholders
+
+      def placeholder_value(placeholder_in_query)
+        placeholder = placeholder_in_query[PLACEHOLDER_NAME_REGEX, 1]
+        placeholders.fetch(placeholder.to_sym)
+      end
+
+      def prepared_placeholder_value(value)
+        return value unless value.is_a?(Array)
+
+        Quoting.quote(value)
+      end
+    end
+  end
+end

data/lib/click_house/client/query_builder.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+require 'active_record'
+
+module ClickHouse
+  module Client
+    class QueryBuilder < QueryLike
+      attr_reader :table
+      attr_accessor :conditions, :manager
+
+      VALID_NODES = [
+        Arel::Nodes::In,
+        Arel::Nodes::Equality,
+        Arel::Nodes::LessThan,
+        Arel::Nodes::LessThanOrEqual,
+        Arel::Nodes::GreaterThan,
+        Arel::Nodes::GreaterThanOrEqual,
+        Arel::Nodes::NamedFunction,
+        Arel::Nodes::NotIn,
+        Arel::Nodes::NotEqual,
+        Arel::Nodes::Between,
+        Arel::Nodes::And,
+        Arel::Nodes::Or,
+        Arel::Nodes::Grouping
+      ].freeze
+
+      def initialize(table_name)
+        @table = Arel::Table.new(table_name)
+        @manager = Arel::SelectManager.new(Arel::Table.engine).from(@table).project(Arel.star)
+        @conditions = []
+      end
+
+      # The `where` method currently only supports IN and equal to queries along
+      # with above listed VALID_NODES.
+      # For example, using a range (start_date..end_date) will result in incorrect SQL.
+      # If you need to query a range, use greater than and less than conditions with Arel.
+      #
+      # Correct usage:
+      #   query.where(query.table[:created_at].lteq(Date.today)).to_sql
+      #   "SELECT * FROM \"table\" WHERE \"table\".\"created_at\" <= '2023-08-01'"
+      #
+      # This also supports array conditions which will result in an IN query.
+      #   query.where(entity_id: [1,2,3]).to_sql
+      #   "SELECT * FROM \"table\" WHERE \"table\".\"entity_id\" IN (1, 2, 3)"
+      #
+      # Range support and more `Arel::Nodes` could be considered for future iterations.
+      # @return [ClickHouse::QueryBuilder] New instance of query builder.
+      def where(conditions)
+        validate_condition_type!(conditions)
+
+        deep_clone.tap do |new_instance|
+          if conditions.is_a?(Arel::Nodes::Node)
+            new_instance.conditions << conditions
+          else
+            add_conditions_to(new_instance, conditions)
+          end
+        end
+      end
+
+      def select(*fields)
+        deep_clone.tap do |new_instance|
+          existing_fields = new_instance.manager.projections.filter_map do |projection|
+            if projection.is_a?(Arel::Attributes::Attribute)
+              projection.name.to_s
+            elsif projection.to_s == '*'
+              nil
+            end
+          end
+
+          new_projections = (existing_fields + fields).map do |field|
+            if field.is_a?(Symbol)
+              field.to_s
+            else
+              field
+            end
+          end
+
+          new_instance.manager.projections = new_projections.uniq.map do |field|
+            if field.is_a?(Arel::Expressions)
+              field
+            else
+              new_instance.table[field.to_s]
+            end
+          end
+        end
+      end
+
+      def order(field, direction = :asc)
+        validate_order_direction!(direction)
+
+        deep_clone.tap do |new_instance|
+          table_field = new_instance.table[field]
+          new_order = direction.to_s.casecmp('desc').zero? ? table_field.desc : table_field.asc
+          new_instance.manager.order(new_order)
+        end
+      end
+
+      def group(*columns)
+        deep_clone.tap do |new_instance|
+          new_instance.manager.group(*columns)
+        end
+      end
+
+      def limit(count)
+        manager.take(count)
+        self
+      end
+
+      def offset(count)
+        manager.skip(count)
+        self
+      end
+
+      def to_sql
+        apply_conditions!
+
+        visitor = Arel::Visitors::ToSql.new(ClickHouse::Client::ArelEngine.new)
+        visitor.accept(manager.ast, Arel::Collectors::SQLString.new).value
+      end
+
+      def to_redacted_sql(bind_index_manager = ClickHouse::Client::BindIndexManager.new)
+        ClickHouse::Client::Redactor.redact(self, bind_index_manager)
+      end
+
+      private
+
+      def validate_condition_type!(condition)
+        return unless condition.is_a?(Arel::Nodes::Node) && VALID_NODES.exclude?(condition.class)
+
+        raise ArgumentError, "Unsupported Arel node type for QueryBuilder: #{condition.class.name}"
+      end
+
+      def add_conditions_to(instance, conditions)
+        conditions.each do |key, value|
+          instance.conditions << if value.is_a?(Array)
+                                   instance.table[key].in(value)
+                                 else
+                                   instance.table[key].eq(value)
+                                 end
+        end
+      end
+
+      def deep_clone
+        self.class.new(table.name).tap do |new_instance|
+          new_instance.manager = manager.clone
+          new_instance.conditions = conditions.map(&:clone)
+        end
+      end
+
+      def apply_conditions!
+        manager.constraints.clear
+        conditions.each { |condition| manager.where(condition) }
+      end
+
+      def validate_order_direction!(direction)
+        return if %w[asc desc].include?(direction.to_s.downcase)
+
+        raise ArgumentError, "Invalid order direction '#{direction}'. Must be :asc or :desc"
+      end
+    end
+  end
+end

data/lib/click_house/client/query_like.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Client
+    class QueryLike
+      # Build a SQL string that can be executed on a ClickHouse database.
+      def to_sql
+        raise NotImplementedError
+      end
+
+      # Redacted version of the SQL query generated by the to_sql method where the
+      # placeholders are stripped. These queries are meant to be exported to external
+      # log aggregation systems.
+      def to_redacted_sql(bind_index_manager = BindIndexManager.new)
+        raise NotImplementedError
+      end
+
+      # Override when placeholders should be supported
+      def prepared_placeholders
+        {}
+      end
+
+      private
+
+      # Override when placeholders should be supported
+      def placeholders
+        {}
+      end
+    end
+  end
+end

data/lib/click_house/client/quoting.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Client
+    module Quoting
+      class << self
+        def quote(value)
+          case value
+          when Numeric then value.to_s
+          when String, Symbol then "'#{value.to_s.gsub('\\', '\&\&').gsub("'", "''")}'"
+          when Array then "[#{value.map { |v| quote(v) }.join(',')}]"
+          when nil then "NULL"
+          else quote_str(value.to_s)
+          end
+        end
+
+        private
+
+        def quote_str(value)
+          "'#{value.gsub("'", "''")}'"
+        end
+      end
+    end
+  end
+end

data/lib/click_house/client/redactor.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Client
+    module Redactor
+      # Redacts the SQL query represented by the query builder.
+      #
+      # @param query_builder [::ClickHouse::Querybuilder] The query builder object to be redacted.
+      # @return [String] The redacted SQL query as a string.
+      # @raise [ArgumentError] when the condition in the query is of an unsupported type.
+      #
+      # Example:
+      #   query_builder = ClickHouse::QueryBuilder.new('users').where(name: 'John Doe')
+      #   redacted_query = ClickHouse::Redactor.redact(query_builder)
+      #   # The redacted_query will contain the SQL query with values replaced by placeholders.
+      #   output: "SELECT * FROM \"users\" WHERE \"users\".\"name\" = $1"
+      def self.redact(query_builder, bind_manager = ClickHouse::Client::BindIndexManager.new)
+        cloned_query_builder = query_builder.clone
+
+        cloned_query_builder.conditions = cloned_query_builder.conditions.map do |condition|
+          redact_condition(condition, bind_manager)
+        end
+
+        cloned_query_builder.manager.constraints.clear
+        cloned_query_builder.conditions.each do |condition|
+          cloned_query_builder.manager.where(condition)
+        end
+
+        visitor = Arel::Visitors::ToSql.new(ClickHouse::Client::ArelEngine.new)
+        visitor.accept(cloned_query_builder.manager.ast, Arel::Collectors::SQLString.new).value
+      end
+
+      def self.redact_condition(condition, bind_manager)
+        case condition
+        when Arel::Nodes::In
+          condition.left.in(Array.new(condition.right.size) { Arel.sql(bind_manager.next_bind_str) })
+        when Arel::Nodes::Equality
+          condition.left.eq(Arel.sql(bind_manager.next_bind_str))
+        when Arel::Nodes::LessThan
+          condition.left.lt(Arel.sql(bind_manager.next_bind_str))
+        when Arel::Nodes::LessThanOrEqual
+          condition.left.lteq(Arel.sql(bind_manager.next_bind_str))
+        when Arel::Nodes::GreaterThan
+          condition.left.gt(Arel.sql(bind_manager.next_bind_str))
+        when Arel::Nodes::GreaterThanOrEqual
+          condition.left.gteq(Arel.sql(bind_manager.next_bind_str))
+        when Arel::Nodes::NamedFunction
+          redact_named_function(condition, bind_manager)
+        else
+          raise ArgumentError, "Unsupported Arel node type for Redactor: #{condition.class}"
+        end
+      end
+
+      def self.redact_named_function(condition, bind_manager)
+        redacted_condition =
+          Arel::Nodes::NamedFunction.new(condition.name, condition.expressions.dup)
+
+        case redacted_condition.name
+        when 'startsWith'
+          redacted_condition.expressions[1] = Arel.sql(bind_manager.next_bind_str)
+        else
+          redacted_condition.expressions = redacted_condition.expressions.map { Arel.sql(bind_manager.next_bind_str) }
+        end
+
+        redacted_condition
+      end
+    end
+  end
+end

data/lib/click_house/client/response.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Client
+    class Response
+      attr_reader :body, :headers
+
+      def initialize(body, http_status_code, headers = {})
+        @body = body
+        @http_status_code = http_status_code
+        @headers = headers
+      end
+
+      def success?
+        @http_status_code == 200
+      end
+    end
+  end
+end

data/lib/click_house/client/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module ClickHouse
+  module Client
+    VERSION = "0.3.2"
+  end
+end