clickhouse-ruby 0.1.0 → 0.2.0

data/lib/clickhouse_ruby/retry_handler.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module ClickhouseRuby
+  # Implements retry logic with exponential backoff and jitter
+  #
+  # This class handles transient failures in ClickHouse connections by
+  # retrying with an exponential backoff strategy and optional jitter.
+  #
+  # @example Basic usage
+  #   handler = RetryHandler.new(max_attempts: 3)
+  #   result = handler.with_retry do
+  #     client.execute('SELECT * FROM users')
+  #   end
+  #
+  # @example With idempotency flag
+  #   handler.with_retry(idempotent: false) do |query_id|
+  #     client.insert('events', data, settings: { query_id: query_id })
+  #   end
+  class RetryHandler
+    # Errors that should trigger a retry
+    RETRIABLE_ERRORS = [
+      ConnectionError,
+      ConnectionTimeout,
+      ConnectionNotEstablished,
+      PoolTimeout,
+    ].freeze
+
+    # HTTP status codes that should trigger a retry
+    RETRIABLE_HTTP_CODES = %w[500 502 503 504 429].freeze
+
+    # @return [Integer] maximum number of attempts
+    attr_reader :max_attempts
+
+    # @return [Float] initial backoff delay in seconds
+    attr_reader :initial_backoff
+
+    # @return [Float] maximum backoff delay in seconds
+    attr_reader :max_backoff
+
+    # @return [Float] exponential backoff multiplier
+    attr_reader :multiplier
+
+    # @return [Symbol] jitter strategy (:full, :equal, or :none)
+    attr_reader :jitter
+
+    # Creates a new RetryHandler
+    #
+    # @param max_attempts [Integer] maximum retry attempts (default: 3)
+    # @param initial_backoff [Float] initial backoff in seconds (default: 1.0)
+    # @param max_backoff [Float] maximum backoff in seconds (default: 120.0)
+    # @param multiplier [Float] backoff multiplier (default: 1.6)
+    # @param jitter [Symbol] jitter strategy (default: :equal)
+    def initialize(
+      max_attempts: 3,
+      initial_backoff: 1.0,
+      max_backoff: 120.0,
+      multiplier: 1.6,
+      jitter: :equal
+    )
+      @max_attempts = max_attempts
+      @initial_backoff = initial_backoff
+      @max_backoff = max_backoff
+      @multiplier = multiplier
+      @jitter = jitter
+    end
+
+    # Executes a block with retry logic
+    #
+    # Yields to the block with an optional query_id. If the block raises
+    # a retriable error, retries with exponential backoff up to max_attempts.
+    # Non-retriable errors are re-raised immediately.
+    #
+    # @param idempotent [Boolean] whether the operation is idempotent (default: true)
+    # @param query_id [String, nil] query ID for deduplication (optional)
+    # @yieldparam query_id [String] generated or provided query_id
+    # @return [Object] return value from the block
+    # @raise [Error] if all retries are exhausted or error is non-retriable
+    #
+    # @example Idempotent operation (SELECT)
+    #   handler.with_retry { client.execute('SELECT * FROM users') }
+    #
+    # @example Non-idempotent operation (INSERT)
+    #   handler.with_retry(idempotent: false) do |qid|
+    #     client.insert('events', data, settings: { query_id: qid })
+    #   end
+    def with_retry(idempotent: true, query_id: nil)
+      attempts = 0
+      generated_query_id = query_id || SecureRandom.uuid
+
+      begin
+        attempts += 1
+        yield(generated_query_id)
+      rescue *RETRIABLE_ERRORS => e
+        handle_retry(attempts, e, idempotent)
+        retry
+      rescue QueryError => e
+        # Check if HTTP code is retriable (server error or rate limit)
+        if retriable_http_error?(e)
+          handle_retry(attempts, e, idempotent)
+          retry
+        end
+        raise
+      end
+    end
+
+    # Checks if an error is retriable
+    #
+    # @param error [Exception] the error to check
+    # @return [Boolean] true if the error should trigger a retry
+    def retriable?(error)
+      RETRIABLE_ERRORS.any? { |klass| error.is_a?(klass) } ||
+        retriable_http_error?(error)
+    end
+
+    private
+
+    # Handles retry logic for an attempt
+    #
+    # @param attempts [Integer] number of attempts so far
+    # @param error [Exception] the error that occurred
+    # @param idempotent [Boolean] whether operation is idempotent
+    # @raise [Exception] if max attempts reached
+    # @return [void]
+    def handle_retry(attempts, error, idempotent)
+      raise error if attempts >= @max_attempts
+
+      warn "Retrying non-idempotent operation - possible duplicates" unless idempotent
+
+      delay = calculate_delay(attempts)
+      sleep(delay)
+    end
+
+    # Calculates backoff delay with jitter
+    #
+    # Implements exponential backoff:
+    #   base = initial_backoff * (multiplier ^ (attempt - 1))
+    #   capped = min(base, max_backoff)
+    #
+    # Then applies jitter strategy:
+    # - :full - random(0, capped)
+    # - :equal - capped/2 + random(0, capped/2)
+    # - :none - capped (no jitter)
+    #
+    # @param attempt [Integer] the attempt number (1-based)
+    # @return [Float] delay in seconds
+    def calculate_delay(attempt)
+      base = @initial_backoff * (@multiplier**(attempt - 1))
+      capped = [base, @max_backoff].min
+
+      case @jitter
+      when :full
+        rand * capped
+      when :none
+        capped
+      else
+        # Default to equal jitter (both :equal and unknown values)
+        (capped / 2.0) + (rand * (capped / 2.0))
+      end
+    end
+
+    # Checks if an error with HTTP status is retriable
+    #
+    # @param error [Exception] the error to check
+    # @return [Boolean] true if HTTP status indicates retriable error
+    def retriable_http_error?(error)
+      error.respond_to?(:http_status) &&
+        RETRIABLE_HTTP_CODES.include?(error.http_status.to_s)
+    end
+  end
+end

data/lib/clickhouse_ruby/streaming_result.rb

@@ -0,0 +1,309 @@
+# frozen_string_literal: true
+
+require "json"
+require "uri"
+require "net/http"
+require "zlib"
+
+module ClickhouseRuby
+  # Memory-efficient streaming result for large queries
+  #
+  # StreamingResult provides an Enumerable interface for processing ClickHouse
+  # query results without loading all rows into memory. Rows are parsed
+  # line-by-line as they arrive from the server.
+  #
+  # Features:
+  # - Enumerable interface for chainable operations
+  # - Lazy evaluation (no data loaded until iterated)
+  # - Support for gzip decompression
+  # - Progress callbacks
+  # - Batch processing
+  #
+  # @example Basic usage
+  #   client.stream_execute('SELECT * FROM huge_table').each do |row|
+  #     process(row)
+  #   end
+  #
+  # @example Lazy enumeration with filtering
+  #   result = client.stream_execute('SELECT * FROM huge_table')
+  #     .lazy
+  #     .select { |row| row['active'] == 1 }
+  #     .take(100)
+  #     .to_a
+  #
+  # @example Batch processing
+  #   result.each_batch(size: 1000) do |batch|
+  #     insert_into_cache(batch)
+  #   end
+  #
+  class StreamingResult
+    include Enumerable
+
+    # Creates a new streaming result
+    #
+    # @param connection [Connection] the ClickHouse connection
+    # @param sql [String] the SQL query to execute
+    # @param format [String] response format (default: JSONEachRow)
+    # @param compression [String, nil] compression algorithm ('gzip' or nil)
+    def initialize(connection, sql, format: "JSONEachRow", compression: nil)
+      @connection = connection
+      @sql = sql
+      @format = format
+      @compression = compression
+      @progress_callback = nil
+    end
+
+    # Sets a callback for progress updates
+    #
+    # ClickHouse sends progress headers during execution:
+    # X-ClickHouse-Progress: {"read_rows":"1000","read_bytes":"50000"}
+    #
+    # @yield [Hash] progress data
+    # @return [self] for method chaining
+    #
+    # @example
+    #   result.on_progress do |progress|
+    #     puts "Processed #{progress['read_rows']} rows"
+    #   end.each { |row| ... }
+    def on_progress(&block)
+      @progress_callback = block
+      self
+    end
+
+    # Iterates over each row in the result
+    #
+    # Returns an Enumerator if no block is given, allowing for lazy evaluation.
+    #
+    # @yield [Hash] each row as a parsed JSON object
+    # @return [Enumerator] if no block given, otherwise nil
+    #
+    # @example With block
+    #   result.each { |row| puts row['id'] }
+    #
+    # @example Returns enumerator without block
+    #   enumerator = result.each
+    def each(&block) # rubocop:disable Naming/BlockForwarding
+      return enum_for(__method__) unless block_given?
+
+      stream_query(&block) # rubocop:disable Naming/BlockForwarding
+    end
+
+    # Iterates over rows in batches
+    #
+    # Useful for batch processing (e.g., bulk database operations).
+    # The final batch may be smaller than the specified size.
+    #
+    # @param size [Integer] batch size (default: 1000)
+    # @yield [Array<Hash>] each batch of rows
+    # @return [Enumerator] if no block given, otherwise nil
+    #
+    # @example
+    #   result.each_batch(size: 500) do |batch|
+    #     insert_batch(batch)
+    #   end
+    def each_batch(size: 1000)
+      return enum_for(__method__, size: size) unless block_given?
+
+      batch = []
+      each do |row|
+        batch << row
+        if batch.size >= size
+          yield batch
+          batch = []
+        end
+      end
+      yield batch if batch.any?
+    end
+
+    private
+
+    # Executes the streaming query
+    #
+    # @yield [Hash] each parsed row
+    # @return [void]
+    def stream_query(&block) # rubocop:disable Naming/BlockForwarding
+      uri = build_uri
+      request = build_request(uri)
+
+      Net::HTTP.start(
+        uri.host,
+        uri.port,
+        use_ssl: @connection.use_ssl,
+        ssl_version: :TLSv1_2, # rubocop:disable Naming/VariableNumber
+      ) do |http|
+        http.request(request) do |response|
+          check_response_status(response)
+          handle_progress(response)
+
+          parse_streaming_body(response, &block) # rubocop:disable Naming/BlockForwarding
+        end
+      end
+    end
+
+    # Builds the request URI
+    #
+    # @return [URI] the request URI with query parameters
+    def build_uri
+      uri = URI("http#{"s" if @connection.use_ssl}://#{@connection.host}:#{@connection.port}/")
+      params = {
+        "database" => @connection.database,
+        "query" => "#{@sql} FORMAT #{@format}",
+      }
+      params["enable_http_compression"] = "1" if @compression
+      uri.query = URI.encode_www_form(params)
+      uri
+    end
+
+    # Builds the HTTP request
+    #
+    # @param uri [URI] the request URI
+    # @return [Net::HTTP::Get] the HTTP request
+    def build_request(uri)
+      request = Net::HTTP::Get.new(uri)
+      request["Accept-Encoding"] = "gzip" if @compression
+      request
+    end
+
+    # Checks the HTTP response status
+    #
+    # @param response [Net::HTTPResponse] the HTTP response
+    # @return [void]
+    # @raise [QueryError] if status is not 200
+    def check_response_status(response)
+      return if response.code == "200"
+
+      body = response.body || ""
+      raise_clickhouse_error(response, body)
+    end
+
+    # Handles progress header if callback is set
+    #
+    # @param response [Net::HTTPResponse] the HTTP response
+    # @return [void]
+    def handle_progress(response)
+      return unless @progress_callback
+
+      progress = response["X-ClickHouse-Progress"]
+      return unless progress
+
+      @progress_callback.call(JSON.parse(progress))
+    end
+
+    # Parses the streaming response body
+    #
+    # Handles:
+    # - Chunked transfer encoding
+    # - Incomplete line buffering
+    # - Gzip decompression if enabled
+    #
+    # @param response [Net::HTTPResponse] the HTTP response
+    # @yield [Hash] each parsed row
+    # @return [void]
+    def parse_streaming_body(response) # rubocop:disable Metrics/CyclomaticComplexity
+      decompressor = create_decompressor(response)
+      buffer = ""
+
+      response.read_body do |chunk|
+        data = decompressor ? decompressor.inflate(chunk) : chunk
+        buffer += data
+
+        # Process complete lines
+        while buffer.include?("\n")
+          line, buffer = buffer.split("\n", 2)
+          next if line.empty?
+
+          row = parse_row(line)
+          yield row if row
+        end
+      end
+
+      # Finalize decompression
+      if decompressor
+        buffer += decompressor.finish
+        decompressor.close
+      end
+
+      # Process remaining buffer (last line may not end with \n)
+      return if buffer.strip.empty?
+
+      row = parse_row(buffer)
+      yield row if row
+    end
+
+    # Creates a decompressor based on Content-Encoding header
+    #
+    # @param response [Net::HTTPResponse] the HTTP response
+    # @return [Zlib::Inflate, nil] decompressor or nil
+    def create_decompressor(response)
+      case response["Content-Encoding"]
+      when "gzip"
+        Zlib::Inflate.new(16 + Zlib::MAX_WBITS)
+      end
+    end
+
+    # Parses a single row from JSON
+    #
+    # @param line [String] JSON line
+    # @return [Hash, nil] parsed row or nil
+    # @raise [QueryError] if line contains error
+    def parse_row(line)
+      data = JSON.parse(line)
+
+      # Check for error in stream
+      if data["exception"]
+        raise QueryError.new(
+          data["exception"]["message"],
+          code: data["exception"]["code"],
+        )
+      end
+
+      data
+    rescue JSON::ParserError => e
+      raise QueryError.new(
+        "Failed to parse streaming response: #{e.message}",
+        original_error: e,
+      )
+    end
+
+    # Raises a ClickHouse error
+    #
+    # @param response [Net::HTTPResponse] the HTTP response
+    # @param body [String] response body
+    # @return [void]
+    # @raise [QueryError] always raises
+    def raise_clickhouse_error(response, body)
+      code = extract_error_code(body)
+      message = extract_error_message(body)
+
+      error_class = ClickhouseRuby.error_class_for_code(code)
+
+      raise error_class.new(
+        message,
+        code: code,
+        http_status: response.code,
+        sql: @sql,
+      )
+    end
+
+    # Extracts ClickHouse error code from response body
+    #
+    # @param body [String] response body
+    # @return [Integer, nil] error code or nil
+    def extract_error_code(body)
+      match = body.match(/Code:\s*(\d+)/)
+      match ? match[1].to_i : nil
+    end
+
+    # Extracts error message from response body
+    #
+    # @param body [String] response body
+    # @return [String] error message
+    def extract_error_message(body)
+      if body =~ /DB::Exception:\s*(.+?)(?:\s*\(version|$)/m
+        ::Regexp.last_match(1).strip
+      else
+        body.strip.empty? ? "Unknown ClickHouse error" : body.strip
+      end
+    end
+  end
+end

data/lib/clickhouse_ruby/types/array.rb

@@ -20,7 +20,7 @@ module ClickhouseRuby
       # @param element_type [Base] the element type
       def initialize(name, element_type: nil)
         super(name)
-        @element_type = element_type || Base.new('String')
+        @element_type = element_type || Base.new("String")
       end
 
       # Converts a Ruby value to an array
@@ -37,12 +37,7 @@ module ClickhouseRuby
               when ::String
                 parse_array_string(value)
               else
-                raise TypeCastError.new(
-                  "Cannot cast #{value.class} to Array",
-                  from_type: value.class.name,
-                  to_type: to_s,
-                  value: value
-                )
+                raise_cast_error(value, "Cannot cast #{value.class} to Array")
               end
 
         arr.map { |v| @element_type.cast(v) }
@@ -72,10 +67,10 @@ module ClickhouseRuby
       # @param value [Array, nil] the value to serialize
       # @return [String] the SQL literal
       def serialize(value)
-        return 'NULL' if value.nil?
+        return "NULL" if value.nil?
 
         elements = value.map { |v| @element_type.serialize(v) }
-        "[#{elements.join(', ')}]"
+        "[#{elements.join(", ")}]"
       end
 
       # Returns the full type string including element type
@@ -95,17 +90,10 @@ module ClickhouseRuby
         stripped = value.strip
 
         # Handle empty array
-        return [] if stripped == '[]'
+        return [] if stripped == "[]"
 
         # Remove outer brackets
-        unless stripped.start_with?('[') && stripped.end_with?(']')
-          raise TypeCastError.new(
-            "Invalid array format: '#{value}'",
-            from_type: 'String',
-            to_type: to_s,
-            value: value
-          )
-        end
+        raise_format_error(value, "array") unless stripped.start_with?("[") && stripped.end_with?("]")
 
         inner = stripped[1...-1]
         return [] if inner.strip.empty?
@@ -119,48 +107,9 @@ module ClickhouseRuby
       # @param str [String] the inner array string
       # @return [Array] the parsed elements
       def parse_elements(str)
-        elements = []
-        current = ''
-        depth = 0
-        in_string = false
-        escape_next = false
-
-        str.each_char do |char|
-          if escape_next
-            current += char
-            escape_next = false
-            next
-          end
-
-          case char
-          when '\\'
-            escape_next = true
-            current += char
-          when "'"
-            in_string = !in_string
-            current += char
-          when '[', '('
-            depth += 1 unless in_string
-            current += char
-          when ']', ')'
-            depth -= 1 unless in_string
-            current += char
-          when ','
-            if depth.zero? && !in_string
-              elements << parse_element(current.strip)
-              current = ''
-            else
-              current += char
-            end
-          else
-            current += char
-          end
+        StringParser.parse_delimited(str, open_brackets: ["[", "("], close_brackets: ["]", ")"]).map do |el|
+          parse_element(el)
         end
-
-        # Don't forget the last element
-        elements << parse_element(current.strip) unless current.strip.empty?
-
-        elements
       end
 
       # Parses a single element, removing quotes if necessary
@@ -168,12 +117,10 @@ module ClickhouseRuby
       # @param str [String] the element string
       # @return [Object] the parsed element
       def parse_element(str)
-        # Remove surrounding quotes if present
+        # Unquote strings (e.g., "'[test]'" -> "[test]")
+        # Preserve unquoted values like nested arrays [1, 2] or numbers
         if str.start_with?("'") && str.end_with?("'")
-          str[1...-1].gsub("\\'", "'")
-        elsif str.start_with?('[')
-          # Nested array - let the element type handle it
-          str
+          StringParser.unquote(str)
         else
           str
         end

data/lib/clickhouse_ruby/types/base.rb

@@ -72,6 +72,65 @@ module ClickhouseRuby
       def hash
         name.hash
       end
+
+      protected
+
+      # Raises a TypeCastError for unsupported type conversion
+      #
+      # @param value [Object] the value that could not be cast
+      # @param message [String, nil] optional custom message
+      # @raise [TypeCastError] always
+      def raise_cast_error(value, message = nil)
+        msg = message || "Cannot cast #{value.class} to #{name}"
+        raise TypeCastError.new(
+          msg,
+          from_type: value.class.name,
+          to_type: name,
+          value: value,
+        )
+      end
+
+      # Raises a TypeCastError for invalid string format
+      #
+      # @param value [String] the invalid string
+      # @param format_name [String] description of expected format
+      # @raise [TypeCastError] always
+      def raise_format_error(value, format_name)
+        raise TypeCastError.new(
+          "Invalid #{format_name} format: '#{value}'",
+          from_type: "String",
+          to_type: name,
+          value: value,
+        )
+      end
+
+      # Raises a TypeCastError for empty string input
+      #
+      # @param value [String] the empty string
+      # @raise [TypeCastError] always
+      def raise_empty_string_error(value)
+        raise TypeCastError.new(
+          "Cannot cast empty string to #{name}",
+          from_type: "String",
+          to_type: name,
+          value: value,
+        )
+      end
+
+      # Raises a TypeCastError for value out of range
+      #
+      # @param value [Numeric] the out-of-range value
+      # @param min [Numeric] minimum allowed value
+      # @param max [Numeric] maximum allowed value
+      # @raise [TypeCastError] always
+      def raise_range_error(value, min, max)
+        raise TypeCastError.new(
+          "Value #{value} is out of range for #{name} (#{min}..#{max})",
+          from_type: value.class.name,
+          to_type: name,
+          value: value,
+        )
+      end
     end
   end
 end