ch_connect 0.2.2 → 0.3.0

data/ext/ch_connect_native/extconf.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+abort "ch_connect requires MRI Ruby" unless RUBY_ENGINE == "ruby"
+
+require "mkmf"
+
+vendor_dir = File.expand_path("../../vendor/clickhouse-c", __dir__)
+$INCFLAGS << " -I#{vendor_dir}"
+
+append_cflags("-O2")
+append_cflags("-std=c11")
+
+# Help mkmf find Homebrew-installed libraries on macOS.
+if RUBY_PLATFORM.include?("darwin")
+  %w[lz4 zstd].each do |pkg|
+    prefix = `brew --prefix #{pkg} 2>/dev/null`.strip
+    next if prefix.empty? || !File.directory?(prefix)
+
+    $INCFLAGS << " -I#{prefix}/include"
+    $LDFLAGS << " -L#{prefix}/lib"
+  end
+end
+
+have_lz4 = have_header("lz4.h") && have_library("lz4", "LZ4_decompress_safe")
+have_zstd = have_header("zstd.h") && have_library("zstd", "ZSTD_decompress")
+
+$defs << (have_lz4 ? "-DCHC_EXT_HAVE_LZ4=1" : "-DCHC_NO_LZ4")
+$defs << (have_zstd ? "-DCHC_EXT_HAVE_ZSTD=1" : "-DCHC_NO_ZSTD")
+
+message "ch_connect_native: lz4=#{have_lz4} zstd=#{have_zstd} (TLS is handled in Ruby via openssl stdlib)\n"
+
+create_makefile("ch_connect/ch_connect_native")

data/lib/ch_connect/config.rb

@@ -10,26 +10,37 @@ module ChConnect
   #
   # @example Using URL
   #   config = ChConnect::Config.new
-  #   config.url = "http://user:pass@localhost:8123/mydb"
+  #   config.url = "clickhouse://user:pass@localhost:9000/mydb"
   class Config
+    URL_SCHEMES = {
+      "clickhouse" => false,
+      "tcp" => false,
+      "clickhouses" => true,
+      "tcps" => true
+    }.freeze
+
     DEFAULTS = {
-      scheme: "http",
       host: "localhost",
-      port: 8123,
+      port: nil,
+      compression: :lz4,
+      ssl: false,
+      ssl_verify: true,
+      ssl_ca: nil,
       database: "default",
-      username: "",
+      username: "default",
       password: "",
       connection_timeout: 5,
       read_timeout: 60,
       write_timeout: 60,
-      keep_alive_timeout: 8,
+      keep_alive_timeout: 60,
       pool_size: 100,
       pool_timeout: 5,
       max_retries: 3,
+      retry_base_interval: 0.05,
+      retry_max_interval: 1.0,
       instrumenter: NullInstrumenter.new
     }.freeze
 
-    # @return [String] URL scheme (http or https)
     # @return [String] ClickHouse server hostname
     # @return [Integer] ClickHouse server port
     # @return [String] Database name
@@ -38,47 +49,77 @@ module ChConnect
     # @return [Integer] Connection timeout in seconds
     # @return [Integer] Read timeout in seconds
     # @return [Integer] Write timeout in seconds
-    # @return [Integer] Keep-alive timeout for idle persistent connections in seconds
+    # @return [Numeric, nil] Maximum pooled connection idle time in seconds
     # @return [Integer] Connection pool size
     # @return [Integer] Pool checkout timeout in seconds
-    # @return [Integer] Max retry attempts on connection errors
+    # @return [Integer] Max retries for establishment failures and opted-in
+    #   idempotent query transport failures
+    # @return [Numeric] Initial retry delay in seconds; zero disables backoff
+    # @return [Numeric] Maximum retry delay in seconds
     # @return [#instrument] Instrumenter for query instrumentation
-    attr_accessor :scheme, :host, :port, :database, :username, :password, :connection_timeout, :read_timeout, :write_timeout, :keep_alive_timeout, :pool_size, :pool_timeout, :max_retries, :instrumenter
+    # @return [Integer] Native protocol port
+    # @return [Symbol, nil] Block compression: :lz4 (default), :zstd or nil
+    # @return [Boolean] Use TLS (default: false)
+    # @return [Boolean] Verify the server certificate when ssl is enabled (default: true)
+    # @return [String, nil] Path to a CA certificate file for TLS verification (default: system CA store)
+    attr_accessor :compression, :ssl, :ssl_verify, :ssl_ca, :host, :database, :username, :password, :connection_timeout, :read_timeout, :write_timeout, :keep_alive_timeout, :pool_size, :pool_timeout, :max_retries, :retry_base_interval, :retry_max_interval, :instrumenter
+    attr_writer :port
 
     # Creates a new configuration instance.
     #
     # @param params [Hash] configuration options
-    # @option params [String] :scheme URL scheme (default: "http")
     # @option params [String] :host server hostname (default: "localhost")
-    # @option params [Integer] :port server port (default: 8123)
+    # @option params [Integer, nil] :port native endpoint port (default: 9000, or 9440 with TLS)
     # @option params [String] :database database name (default: "default")
-    # @option params [String] :username authentication username (default: "")
+    # @option params [String] :username authentication username (default: "default")
     # @option params [String] :password authentication password (default: "")
     # @option params [Integer] :connection_timeout connection timeout in seconds (default: 5)
     # @option params [Integer] :read_timeout read timeout in seconds (default: 60)
     # @option params [Integer] :write_timeout write timeout in seconds (default: 60)
-    # @option params [Integer] :keep_alive_timeout idle persistent connection timeout in seconds (default: 8)
+    # @option params [Numeric, nil] :keep_alive_timeout pooled TCP connection idle timeout (default: 60)
     # @option params [Integer] :pool_size connection pool size (default: 100)
     # @option params [Integer] :pool_timeout pool checkout timeout (default: 5)
-    # @option params [Integer] :max_retries max retry attempts on connection errors (default: 3)
+    # @option params [Integer] :max_retries max retries for establishment
+    #   failures and opted-in idempotent query transport failures (default: 3)
+    # @option params [Numeric] :retry_base_interval initial retry delay in
+    #   seconds, with exponential backoff and jitter (default: 0.05)
+    # @option params [Numeric] :retry_max_interval maximum retry delay in
+    #   seconds (default: 1.0)
     def initialize(params = {})
       DEFAULTS.merge(params).each do |key, value|
         send("#{key}=", value)
       end
     end
 
+    # Returns the explicitly configured port or the default for the active
+    # TLS mode.
+    def port
+      @port || default_port
+    end
+
     # Sets configuration from a URL string.
     #
     # @param url [String] ClickHouse connection URL
     # @return [void]
     def url=(url)
       uri = URI(url)
-      @scheme = uri.scheme
-      @host = uri.host
+      ssl = URL_SCHEMES.fetch(uri.scheme) do
+        raise ArgumentError, "unsupported ClickHouse URL scheme: #{uri.scheme.inspect}"
+      end
+
+      @ssl = ssl
       @port = uri.port
-      @database = uri.path.delete_prefix("/")
-      @username = uri.user
-      @password = uri.password
+      @host = uri.host
+      database = uri.path.delete_prefix("/")
+      @database = database unless database.empty?
+      @username = uri.user if uri.user
+      @password = uri.password if uri.password
+    end
+
+    private
+
+    def default_port
+      ssl ? 9440 : 9000
     end
   end
 end

data/lib/ch_connect/connection.rb

@@ -1,12 +1,233 @@
 # frozen_string_literal: true
 
+require "connection_pool"
+require "openssl"
+require "socket"
+
 module ChConnect
-  # A single connection to ClickHouse server.
+  # A pooled ClickHouse connection built on the clickhouse-c ioless client:
+  # the C extension is a pure protocol state machine + block decoder, and all
+  # socket I/O, TLS, and timeouts live here in Ruby. Result blocks are parsed
+  # in C.
   #
-  # @example
-  #   conn = ChConnect::Connection.new
-  #   response = conn.query("SELECT * FROM users WHERE id = 1")
+  # Maintains a pool of connections (config.pool_size); the native protocol
+  # handles one query at a time per connection, so concurrent queries each
+  # check out their own connection.
   class Connection
+    # Connection establishment failures are safe to retry because no query
+    # bytes have been sent to the server yet.
+    class EstablishmentError < ConnectionError; end
+    private_constant :EstablishmentError
+
+    COMPRESSION_AVAILABLE = {
+      nil => true,
+      :lz4 => NativeClient::LZ4_AVAILABLE,
+      :zstd => NativeClient::ZSTD_AVAILABLE
+    }.freeze
+
+    READ_CHUNK = 64 * 1024
+    WRITE_CHUNK = 64 * 1024
+    PARAM_ESCAPES = {
+      "\0" => "\\0", "\a" => "\\a", "\b" => "\\b", "\e" => "\\e",
+      "\f" => "\\f", "\n" => "\\n", "\r" => "\\r", "\t" => "\\t",
+      "\v" => "\\v", "'" => "\\'", "\\" => "\\\\"
+    }.freeze
+    PARAM_ESCAPE_PATTERN = /[\0\a\b\e\f\n\r\t\v'\\]/
+    RESERVED_SETTINGS = {
+      "network_compression_method" => "set config.compression instead",
+      "output_format_native_encode_types_in_binary_format" => "required by the native decoder"
+    }.freeze
+
+    # A pooled slot owning one socket + protocol state machine pair.
+    # Connects lazily and replaces the connection when it is broken or was
+    # abandoned mid-query (interrupt, timeout, killed thread) — the C client
+    # tracks both via broken?.
+    # @api private
+    class Slot
+      def initialize(config)
+        @config = config
+        @socket = nil
+        @client = nil
+        @pid = Process.pid
+        @read_buf = String.new(capacity: READ_CHUNK)
+      end
+
+      def query(sql, params, settings)
+        ensure_connected
+
+        @client.send_query(sql, params, settings)
+        flush_output
+        loop do
+          case @client.recv_step
+          when :done then break
+          when :want_read then @client.feed(read_chunk)
+          end
+        end
+        @client.take_result
+      rescue IOError, SystemCallError, SocketError, OpenSSL::SSL::SSLError => e
+        raise ConnectionError, "#{e.class}: #{e.message}"
+      end
+
+      def broken? = !@client || @client.broken? || @socket&.closed?
+
+      def close
+        if @pid != Process.pid && @socket.is_a?(OpenSSL::SSL::SSLSocket)
+          # A fork duplicates the fd. Closing the SSLSocket would send a TLS
+          # close_notify on the parent's live session; close only this process's
+          # raw fd instead.
+          safe_close(@socket.to_io)
+        else
+          safe_close(@socket)
+        end
+        @client&.close
+        @socket = nil
+        @client = nil
+      end
+
+      private
+
+      def ensure_connected
+        if @client
+          raise EstablishmentError, "pooled connection is broken" if broken?
+
+          return
+        end
+
+        establish_connection
+      end
+
+      def establish_connection
+        @socket = connect_socket
+        @client = NativeClient.new(
+          @config.database,
+          @config.username,
+          @config.password,
+          @config.compression
+        )
+        handshake
+      rescue ConnectionError, IOError, SystemCallError, SocketError, OpenSSL::SSL::SSLError => e
+        close
+        raise EstablishmentError, "#{e.class}: #{e.message}"
+      end
+
+      def connect_socket
+        socket = Socket.tcp(@config.host, @config.port, connect_timeout: @config.connection_timeout)
+        wrapped = nil
+        begin
+          socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+          wrapped = @config.ssl ? tls_wrap(socket) : socket
+        ensure
+          # a failed TLS handshake would otherwise leak the raw socket:
+          # @socket is not assigned yet, so discard has nothing to close.
+          # (ensure, not rescue: Thread#kill / Timeout skip rescue clauses)
+          safe_close(socket) if wrapped.nil?
+        end
+        wrapped
+      end
+
+      def tls_wrap(socket)
+        ctx = OpenSSL::SSL::SSLContext.new
+        if @config.ssl_verify
+          ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER
+          ctx.verify_hostname = true
+          if @config.ssl_ca
+            ctx.ca_file = @config.ssl_ca
+          else
+            ctx.cert_store = OpenSSL::X509::Store.new.tap(&:set_default_paths)
+          end
+        else
+          ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        end
+
+        ssl = OpenSSL::SSL::SSLSocket.new(socket, ctx)
+        ssl.hostname = @config.host
+        ssl.sync_close = true
+
+        deadline = monotonic_now + @config.connection_timeout
+        loop do
+          readiness = ssl.connect_nonblock(exception: false)
+          break unless readiness == :wait_readable || readiness == :wait_writable
+
+          wait_or_fail(ssl, readiness, deadline - monotonic_now, "TLS handshake timeout")
+        end
+        ssl
+      end
+
+      def handshake
+        deadline = monotonic_now + @config.connection_timeout
+        loop do
+          state = @client.handshake_step
+          flush_output
+          break if state == :done
+
+          # the handshake is part of connecting: bound it by connection_timeout
+          remaining = deadline - monotonic_now
+          raise ConnectionError, "native handshake timeout" if remaining <= 0
+
+          @client.feed(read_chunk(remaining))
+        end
+      end
+
+      def flush_output
+        deadline = monotonic_now + @config.write_timeout if @config.write_timeout
+        while (out = @client.take_output)
+          offset = 0
+          while offset < out.bytesize
+            if deadline && deadline - monotonic_now <= 0
+              raise ConnectionError, "write timeout"
+            end
+
+            pending = out.byteslice(offset, [WRITE_CHUNK, out.bytesize - offset].min)
+            case (written = @socket.write_nonblock(pending, exception: false))
+            when :wait_readable, :wait_writable
+              remaining = deadline && (deadline - monotonic_now)
+              wait_or_fail(@socket, written, remaining, "write timeout")
+            else
+              raise ConnectionError, "connection closed while writing" if written == 0
+
+              offset += written
+            end
+          end
+        end
+      end
+
+      # Reads into a reusable buffer: feed copies the bytes synchronously,
+      # so the buffer never needs to survive past the next read.
+      def read_chunk(timeout = @config.read_timeout)
+        deadline = monotonic_now + timeout if timeout
+        loop do
+          case (chunk = @socket.read_nonblock(READ_CHUNK, @read_buf, exception: false))
+          when :wait_readable, :wait_writable
+            remaining = deadline && (deadline - monotonic_now)
+            wait_or_fail(@socket, chunk, remaining, "read timeout")
+          when nil
+            raise ConnectionError, "connection closed by server"
+          else
+            return chunk
+          end
+        end
+      end
+
+      # readiness is :wait_readable / :wait_writable — also the names of the
+      # IO wait methods.
+      def wait_or_fail(io, readiness, timeout, message)
+        raise ConnectionError, message if timeout && timeout <= 0
+
+        raise ConnectionError, message unless io.to_io.public_send(readiness, timeout)
+      end
+
+      def monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      def safe_close(io)
+        io&.close
+      rescue IOError, SystemCallError
+        nil
+      end
+    end
+    private_constant :Slot
+
     # @return [Config] the configuration used by this connection
     attr_reader :config
 
@@ -14,22 +235,170 @@ module ChConnect
     #
     # @param config [Config] configuration instance (defaults to global config)
     def initialize(config = ChConnect.config)
-      @config = config
-      @transport = HttpTransport.new(config)
+      @config = config.dup.freeze
+      validate_compression!
+      @codec_name = @config.compression&.to_s
+      @pool = ConnectionPool.new(
+        size: @config.pool_size,
+        timeout: @config.pool_timeout,
+        auto_reload_after_fork: true
+      ) do
+        Slot.new(@config)
+      end
     end
 
-    # Executes a SQL query and returns the response.
+    # Executes and instruments a SQL query.
     #
     # @param sql [String] SQL query to execute
-    # @param options [Hash] query options
-    # @option options [Hash] :params query parameters
-    # @return [Response] query response with rows, columns, and metadata
+    # @param params [Hash, nil] query parameters (name => value)
+    # @param settings [Hash, nil] per-query ClickHouse settings
+    # @param idempotent [Boolean] retry transport failures on a fresh connection
+    #   up to config.max_retries (default: false)
+    # @return [Response] fully parsed response
     # @raise [QueryError] if the query fails
-    def query(sql, options = {})
+    # @raise [ConnectionError] if a connection or transport operation fails
+    def query(sql, params: nil, settings: nil, idempotent: false)
       @config.instrumenter.instrument("query.clickhouse", {sql: sql}) do
-        result = @transport.execute(sql, options)
-        NativeFormatParser.new(result.body).parse.with(summary: result.summary)
+        execute_query(sql, params, settings, idempotent)
+      end
+    end
+
+    # Closes all pooled connections.
+    #
+    # @return [void]
+    def close
+      @pool.shutdown(&:close)
+    end
+
+    private
+
+    def execute_query(sql, user_params, user_settings, idempotent)
+      started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond)
+      params = format_params(user_params)
+      settings = format_settings(user_settings)
+      retries = 0
+      reap_idle_connections
+
+      columns, types, rows, summary = begin
+        @pool.with do |slot|
+          slot.query(sql, params, settings)
+        ensure
+          @pool.discard_current_connection(&:close) if slot.broken?
+        end
+      rescue EstablishmentError
+        raise if retries >= @config.max_retries
+        retries += 1
+        backoff_before_retry(retries)
+        retry
+      rescue ConnectionError
+        raise unless idempotent
+        raise if retries >= @config.max_retries
+        retries += 1
+        backoff_before_retry(retries)
+        retry
+      rescue ConnectionPool::TimeoutError => e
+        raise ConnectionError, "could not obtain a TCP connection from the pool: #{e.message}"
+      end
+      summary[:client_elapsed_ns] = Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond) - started_at
+      Response.new(columns: columns, types: types, rows: rows, summary: summary)
+    end
+
+    def reap_idle_connections
+      timeout = @config.keep_alive_timeout
+      @pool.reap(idle_seconds: timeout, &:close) if timeout
+    end
+
+    def backoff_before_retry(retry_number)
+      base = @config.retry_base_interval
+      return unless base.positive?
+
+      ceiling = [base * (2**(retry_number - 1)), @config.retry_max_interval].min
+      sleep(ceiling * (0.5 + rand * 0.5))
+    end
+
+    def validate_compression!
+      available = COMPRESSION_AVAILABLE.fetch(@config.compression) do
+        raise Error, "unknown compression: #{@config.compression.inspect} (use :lz4, :zstd or nil)"
+      end
+      compression = @config.compression
+      unless available
+        raise Error, "compression = #{compression.inspect} but the extension was built without lib#{compression} (install #{compression} and reinstall the gem, or set config.compression = nil)"
+      end
+    end
+
+    # Converts params into native protocol substitutions. Values go through
+    # Field::restoreFromDump on the server, so everything is sent as a quoted
+    # string and converted by the placeholder type (same convention as
+    # clickhouse-cpp).
+    def format_params(params)
+      return nil if params.nil? || params.empty?
+
+      params.map { |name, value| [name.to_s, quote_param(value)] }
+    end
+
+    def quote_param(value)
+      # Query parameters use Field::restoreFromDump. Its nullable marker is a
+      # quoted, doubly escaped text NULL, matching ClickHouse's native client
+      # protocol (the dump parser consumes the first escape layer).
+      return "'\\\\N'" if value.nil?
+
+      inner_dump = value.is_a?(Array) ? dump_array_value(value) : escape_param_string(value.to_s)
+      outer_dump = inner_dump.gsub(/['\\]/) { |char| "\\#{char}" }
+      "'".b << outer_dump << "'"
+    end
+
+    def dump_array_value(value)
+      case value
+      when nil then "NULL"
+      when Array
+        dump = +"[".b
+        value.each_with_index do |element, index|
+          dump << "," unless index.zero?
+          dump << dump_array_value(element)
+        end
+        dump << "]"
+      when String, Symbol then quote_array_string(value.to_s)
+      when Time, DateTime
+        raise ArgumentError, "#{value.class} array parameters are ambiguous; pass a formatted String"
+      when Date then quote_array_string(value.to_s)
+      when true then "true"
+      when false then "false"
+      when Integer, Float, BigDecimal then value.to_s
+      else
+        raise ArgumentError, "unsupported array parameter element: #{value.class}"
+      end
+    end
+
+    def quote_array_string(value)
+      "'".b << escape_param_string(value) << "'"
+    end
+
+    def escape_param_string(value)
+      value.b.gsub(PARAM_ESCAPE_PATTERN, PARAM_ESCAPES)
+    end
+
+    # Settings travel as name/value strings in the Query packet. The wire's
+    # compression flag is boolean and the server picks the response codec
+    # from network_compression_method, so when compression is on we set its
+    # default to the configured codec.
+    def format_settings(user_settings)
+      settings = []
+      settings << ["network_compression_method", @codec_name] if @codec_name
+
+      user_settings&.each do |key, value|
+        name = key.to_s
+        if (guidance = RESERVED_SETTINGS[name])
+          raise ArgumentError, "setting #{name.inspect} is managed by ch_connect; #{guidance}"
+        end
+        formatted = case value
+        when true then "1"
+        when false then "0"
+        else value.to_s
+        end
+        settings << [name, formatted]
       end
+
+      settings.empty? ? nil : settings
     end
   end
 end

data/lib/ch_connect/response.rb

@@ -11,14 +11,6 @@ module ChConnect
   Response = Data.define(:columns, :types, :rows, :summary) do
     include Enumerable
 
-    # @param columns [Array<Symbol>] column names
-    # @param types [Array<Symbol>] column types
-    # @param rows [Array<Array>] row data
-    # @param summary [Hash, nil] ClickHouse query summary with symbol keys
-    def initialize(columns: [], types: [], rows: [], summary: nil)
-      super
-    end
-
     # Iterates over rows as hashes with symbol keys.
     # @yield [Hash] each row as a hash
     # @return [Enumerator] if no block given

data/lib/ch_connect/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ChConnect
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

data/lib/ch_connect.rb

@@ -3,27 +3,18 @@
 require_relative "ch_connect/version"
 require_relative "ch_connect/null_instrumenter"
 require_relative "ch_connect/config"
-require_relative "ch_connect/transport_result"
-require_relative "ch_connect/http_transport"
-require_relative "ch_connect/connection"
 require_relative "ch_connect/response"
-require_relative "ch_connect/body_reader"
-require_relative "ch_connect/native_format_parser"
 
 # Ruby client for ClickHouse database with Native format support.
 #
 # @example Basic usage
 #   ChConnect.configure do |config|
 #     config.host = "localhost"
-#     config.port = 8123
+#     config.port = 9000
 #   end
 #
 #   conn = ChConnect::Connection.new
 #   response = conn.query("SELECT 1")
-#
-# @example Using connection pool
-#   pool = ChConnect::Pool.new
-#   response = pool.query("SELECT * FROM users")
 module ChConnect
   # Base error class for all ChConnect errors
   class Error < StandardError; end
@@ -31,6 +22,9 @@ module ChConnect
   # Raised when a query fails (syntax error, unknown table, etc.)
   class QueryError < Error; end
 
+  # Raised on network/connection failures
+  class ConnectionError < Error; end
+
   # Raised when encountering an unsupported ClickHouse data type
   class UnsupportedTypeError < Error; end
 
@@ -46,6 +40,15 @@ module ChConnect
   # @yield [Config] the configuration instance
   # @return [void]
   def self.configure
-    yield(config) if block_given?
+    yield config
   end
 end
+
+begin
+  require "ch_connect/ch_connect_native"
+rescue LoadError => e
+  raise ChConnect::Error, "ch_connect requires the compiled native extension: #{e.message}"
+end
+ChConnect.private_constant :NativeClient
+
+require_relative "ch_connect/connection"