pinot-client 1.7.0 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce082ab7ebdfd2003c324e13981a48c4735196a424a98f40ca812cf755846cb5
-  data.tar.gz: 8bb6cc60572a748c9acb0817ccc58fdb4a4bc369b016503392b7326d79d01ecf
+  metadata.gz: ddb501e91483838437691807ff09f98be16584f5b5e9e25c32982581db9fc226
+  data.tar.gz: 69f4df0c9fae29d42388b5fa6e1f55a1c722f70673c91b5ad43339762ad052fe
 SHA512:
-  metadata.gz: df40aa52c3ac2101d8cdaf308b741eb056eb17163beabff92c57b5800047ae570d63e10369fb5778040a2986b1c5b9f959df71ec425f3ffeb9eb49f9366a6c31
-  data.tar.gz: bdfe5f80fdee0c0b9b8b4ce1b216a16f098bc27df369a9a62983aa7a0672866260215cfa5ebacb678bb068d389824812491ae2b537f9abe1ea1d9fdf899fda15
+  metadata.gz: 974f22a9db31c53e6a0534400347df98b419a1f6fffc351594bc6ea06456d211cda4ede80390fdf7c9b67a0f0559a6772a8225cc552b2844fb28e33ea11220b6
+  data.tar.gz: 711718da0cb66cd42457c657f58fc29450049094a1f6bf9012557a3b484dc59ca2d66f45bbf9c2c716597779a084fc59ab7fb15730fbac4648ac47823291d9c4

data/README.md

@@ -62,6 +62,30 @@ The client periodically polls the controller's `/v2/brokers/tables` API and auto
 client = Pinot.from_controller("localhost:9000")
 ```
 
+### Via gRPC
+
+Requires the `grpc` gem (`gem "grpc", "~> 1.65"` in your Gemfile).
+
+```ruby
+grpc_config = Pinot::GrpcConfig.new(
+  broker_list: ["localhost:8090"],
+  timeout:     10,
+  extra_metadata: { "Authorization" => "Bearer <token>" }
+)
+config = Pinot::ClientConfig.new(grpc_config: grpc_config)
+client = Pinot.from_config(config)
+```
+
+### From ZooKeeper (dynamic broker discovery)
+
+Requires the `zk` gem (`gem "zk"` in your Gemfile). Watches `/EXTERNALVIEW/brokerResource` and automatically picks up broker changes.
+
+```ruby
+zk_config = Pinot::ZookeeperConfig.new(zk_path: "localhost:2181")
+config = Pinot::ClientConfig.new(zookeeper_config: zk_config)
+client = Pinot.from_config(config)
+```
+
 ### From a `ClientConfig`
 
 ```ruby
@@ -79,6 +103,8 @@ config = Pinot::ClientConfig.new(
 client = Pinot.from_config(config)
 ```
 
+`validate!` raises `Pinot::ConfigurationError` early if the config is invalid.
+
 ### With TLS
 
 ```ruby
@@ -111,6 +137,20 @@ config = Pinot::ClientConfig.new(
 client = Pinot.from_config(config)
 ```
 
+## Instrumentation
+
+Hook into every query execution for metrics, tracing, or alerting:
+
+```ruby
+Pinot::Instrumentation.on_query = lambda do |event|
+  puts "#{event[:table]} #{event[:query][0..50]} " \
+       "#{event[:duration_ms].round(1)}ms " \
+       "success=#{event[:success]}"
+end
+```
+
+The event hash contains: `:table`, `:query`, `:duration_ms` (Float), `:success` (Boolean), `:error` (Exception or nil).
+
 ## Executing Queries
 
 ### Simple SQL
@@ -148,6 +188,19 @@ config = Pinot::ClientConfig.new(
 client = Pinot.from_config(config)
 ```
 
+### Per-request timeout
+
+```ruby
+# Global default via config
+config = Pinot::ClientConfig.new(
+  broker_list:      ["localhost:8000"],
+  query_timeout_ms: 5000
+)
+
+# One-off override per query
+resp = client.execute_sql_with_timeout("myTable", "SELECT * FROM myTable", 3000)
+```
+
 ### Trace
 
 ```ruby
@@ -182,6 +235,19 @@ rescue Pinot::Error => e
 end
 ```
 
+## Retry
+
+Configure automatic retry with exponential backoff for transient errors (connection reset, timeout, HTTP 503):
+
+```ruby
+config = Pinot::ClientConfig.new(
+  broker_list:       ["localhost:8000"],
+  max_retries:       3,
+  retry_interval_ms: 200   # base interval; doubles each attempt (200ms, 400ms, 800ms)
+)
+client = Pinot.from_config(config)
+```
+
 ## Reading Results
 
 `execute_sql` returns a `Pinot::BrokerResponse`. Results are in `result_table`:

data/lib/pinot/config.rb

@@ -22,7 +22,9 @@ module Pinot
   class ClientConfig
     attr_accessor :broker_list, :http_timeout, :query_timeout_ms, :extra_http_header,
                   :use_multistage_engine, :controller_config, :logger, :tls_config,
-                  :grpc_config, :zookeeper_config
+                  :grpc_config, :zookeeper_config,
+                  :max_retries,      # Integer, default 0 (no retry)
+                  :retry_interval_ms # Integer ms base interval, default 200
 
     def initialize(
       broker_list: [],
@@ -34,7 +36,9 @@ module Pinot
       logger: nil,
       tls_config: nil,
       grpc_config: nil,
-      zookeeper_config: nil
+      zookeeper_config: nil,
+      max_retries: 0,
+      retry_interval_ms: 200
     )
       @broker_list = broker_list
       @http_timeout = http_timeout
@@ -47,6 +51,8 @@ module Pinot
       @grpc_config = grpc_config
       @zookeeper_config = zookeeper_config
       @query_timeout_ms = query_timeout_ms
+      @max_retries = max_retries
+      @retry_interval_ms = retry_interval_ms
     end
 
     def validate!

data/lib/pinot/connection_factory.rb

@@ -37,7 +37,9 @@ module Pinot
       transport = JsonHttpTransport.new(
         http_client: inner,
         extra_headers: config.extra_http_header || {},
-        logger: config.logger
+        logger: config.logger,
+        max_retries: config.max_retries || 0,
+        retry_interval_ms: config.retry_interval_ms || 200
       )
 
       return Connection.new(
@@ -53,7 +55,9 @@ module Pinot
     transport = JsonHttpTransport.new(
       http_client: inner,
       extra_headers: config.extra_http_header || {},
-      logger: config.logger
+      logger: config.logger,
+      max_retries: config.max_retries || 0,
+      retry_interval_ms: config.retry_interval_ms || 200
     )
 
     selector = build_selector(config, inner)

data/lib/pinot/transport.rb

@@ -9,11 +9,14 @@ module Pinot
     MAX_POOL_SIZE = 5
     KEEP_ALIVE_TIMEOUT = 30
 
+    PoolEntry = Struct.new(:http, :checked_in_at)
+
     def initialize(timeout: nil, tls_config: nil)
       @timeout = timeout
       @tls_config = tls_config
       @pool = {}
       @pool_mutex = Mutex.new
+      @reaper = start_reaper
     end
 
     def post(url, body:, headers: {})
@@ -52,20 +55,61 @@ module Pinot
     end
 
     def checkout(key, uri)
-      @pool_mutex.synchronize { @pool[key]&.pop } || new_connection(uri)
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      http = @pool_mutex.synchronize do
+        entries = @pool[key] ||= []
+        fresh = nil
+        while (entry = entries.pop)
+          if now - entry.checked_in_at < KEEP_ALIVE_TIMEOUT
+            fresh = entry.http
+            break
+          else
+            entry.http.finish rescue nil
+          end
+        end
+        fresh
+      end
+      http || new_connection(uri)
     end
 
     def checkin(key, http)
       @pool_mutex.synchronize do
         pool_for_key = @pool[key] ||= []
         if pool_for_key.size < MAX_POOL_SIZE
-          pool_for_key.push(http)
+          pool_for_key.push(PoolEntry.new(http, Process.clock_gettime(Process::CLOCK_MONOTONIC)))
         else
           http.finish rescue nil
         end
       end
     end
 
+    def start_reaper
+      t = Thread.new do
+        loop do
+          sleep KEEP_ALIVE_TIMEOUT / 2.0
+          reap_stale_connections
+        end
+      end
+      t.abort_on_exception = false
+      t
+    end
+
+    def reap_stale_connections
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      @pool_mutex.synchronize do
+        @pool.each_value do |entries|
+          entries.reject! do |entry|
+            if now - entry.checked_in_at >= KEEP_ALIVE_TIMEOUT
+              entry.http.finish rescue nil
+              true
+            else
+              false
+            end
+          end
+        end
+      end
+    end
+
     def new_connection(uri)
       http = Net::HTTP.new(uri.host, uri.port)
       configure_ssl(http, uri)
@@ -108,33 +152,60 @@ module Pinot
       "Content-Type" => "application/json; charset=utf-8"
     }.freeze
 
-    def initialize(http_client:, extra_headers: {}, timeout_ms: nil, logger: nil)
+    RETRYABLE_ERRORS = [
+      Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::ETIMEDOUT,
+      Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout
+    ].freeze
+
+    def initialize(http_client:, extra_headers: {}, timeout_ms: nil, logger: nil,
+                   max_retries: 0, retry_interval_ms: 200)
       @http_client = http_client
       @extra_headers = extra_headers
       @timeout_ms = timeout_ms
       @logger = logger
+      @max_retries = max_retries
+      @retry_interval_ms = retry_interval_ms
     end
 
     def execute(broker_address, request)
       logger.debug "Pinot query to #{broker_address}: #{request.query}"
 
-      url = build_url(broker_address, request.query_format)
-      body = build_body(request)
-      headers = DEFAULT_HEADERS
-        .merge(@extra_headers)
-        .merge("X-Correlation-Id" => SecureRandom.uuid)
+      attempts = 0
+      max_attempts = (@max_retries || 0) + 1
 
-      resp = @http_client.post(url, body: body, headers: headers)
+      begin
+        attempts += 1
 
-      unless resp.code.to_i == 200
-        logger.error "Pinot broker returned HTTP #{resp.code}"
-        raise TransportError, "http exception with HTTP status code #{resp.code}"
-      end
+        url = build_url(broker_address, request.query_format)
+        body = build_body(request)
+        headers = DEFAULT_HEADERS
+          .merge(@extra_headers)
+          .merge("X-Correlation-Id" => SecureRandom.uuid)
 
-      begin
-        BrokerResponse.from_json(resp.body)
-      rescue JSON::ParserError => e
-        raise e.message
+        resp = @http_client.post(url, body: body, headers: headers)
+
+        if resp.code == "503"
+          logger.error "Pinot broker returned HTTP #{resp.code}"
+          raise TransportError, "http exception with HTTP status code #{resp.code}"
+        end
+
+        unless resp.code.to_i == 200
+          logger.error "Pinot broker returned HTTP #{resp.code}"
+          raise TransportError, "http exception with HTTP status code #{resp.code}"
+        end
+
+        begin
+          BrokerResponse.from_json(resp.body)
+        rescue JSON::ParserError => e
+          raise e.message
+        end
+      rescue TransportError, *RETRYABLE_ERRORS => e
+        if attempts < max_attempts
+          sleep_ms = (@retry_interval_ms || 200) * (2 ** (attempts - 1))
+          sleep(sleep_ms / 1000.0)
+          retry
+        end
+        raise
       end
     end
 

data/lib/pinot/version.rb

@@ -1,3 +1,3 @@
 module Pinot
-  VERSION = "1.7.0"
+  VERSION = "1.10.0"
 end

data/lib/pinot.rb

@@ -22,9 +22,9 @@ require_relative "pinot/connection"
 require_relative "pinot/prepared_statement"
 require_relative "pinot/connection_factory"
 
+require_relative "pinot/grpc_config"
 begin
   require_relative "pinot/grpc_transport"
-  require_relative "pinot/grpc_config"
 rescue LoadError
   # grpc gem not available; GrpcTransport disabled
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pinot-client
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.10.0
 platform: ruby
 authors:
 - Xiang Fu