pinot-client 1.26.0 → 1.27.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0c6d47938639a210b77881492e1155d26728be12f987b9721712f16503183ec3
-  data.tar.gz: 9e14c467edbba3ad7fff1e20f6b288ac810895c5069549cd33f08e2972b8d739
+  metadata.gz: dfd22c5e76430105c17c3a1279b002084aebb894d4c561913908a4d00038b519
+  data.tar.gz: e668583cdd1afde9b9b35dab28c46b2e707643911a0800cdb98950ac1de2731b
 SHA512:
-  metadata.gz: 73ec7a9ae84e96f71618f430ed542527fb9f9e16f64b1d0d78a1a8bcb0ba1950f6d7117072f0318ffcc08fdb0df2143be268667d6b96e4914d52fc4f0448c0fa
-  data.tar.gz: 9fe83c991b5ef549ce915dbe4af55b50143d17d6d9660915d5fd6308798541f81068e557317b8b88af167b55a63ea7781ef14dfff428cd51b671df7d2a0267b2
+  metadata.gz: 1aba59c0e39a601dacccfa2f413b9405ad0835e94555272a8f7bf8177b421fd0eac861144fb44d24c69ec41f51834b67a786ed4bd29afb48ae6667671f5ce112
+  data.tar.gz: c81b393e62ffba2390a7c21a52fb0b0a3f0fdfd0421b3a78d9bcbcc772a35f2613e451f58a28cdcb9ecd30c6120aa114f94ae3a80c825594a0cc9baa98a521ca

data/lib/pinot/active_support_notifications.rb

@@ -0,0 +1,81 @@
+require "pinot"
+
+module Pinot
+  # Opt-in ActiveSupport::Notifications bridge for Rails applications.
+  #
+  # == Setup
+  #
+  # Add one line to an initializer (e.g. config/initializers/pinot.rb):
+  #
+  #   require "pinot/active_support_notifications"
+  #   Pinot::ActiveSupportNotifications.install!
+  #
+  # That's it. Every query executed via Connection#execute_sql (including those
+  # from execute_sql_with_params, execute_many, and PreparedStatement) will
+  # publish a "sql.pinot" event on the ActiveSupport::Notifications bus.
+  #
+  # == Subscribing
+  #
+  #   ActiveSupport::Notifications.subscribe("sql.pinot") do |name, start, finish, id, payload|
+  #     Rails.logger.debug "[Pinot] #{payload[:name]} — #{payload[:sql]} (#{payload[:duration].round(1)} ms)"
+  #   end
+  #
+  # == Payload keys
+  #
+  #   :sql              — the SQL query string
+  #   :name             — the Pinot table name (empty string for table-less queries)
+  #   :duration         — execution time in milliseconds (Float)
+  #   :success          — true on success, false when an exception was raised
+  #   :exception        — [ExceptionClassName, message] on error, absent on success
+  #                       (follows the ActiveSupport::Notifications convention)
+  #   :exception_object — the raw exception on error, absent on success
+  #                       (follows the ActiveSupport::Notifications convention)
+  #
+  # == Lifecycle
+  #
+  # The bridge is installed idempotently:
+  #
+  #   Pinot::ActiveSupportNotifications.install!   # register
+  #   Pinot::ActiveSupportNotifications.installed? # => true
+  #   Pinot::ActiveSupportNotifications.uninstall! # deregister (e.g. in tests)
+  #
+  # Note: this gem does NOT depend on activesupport. The bridge requires
+  # ActiveSupport::Notifications to already be defined at install! time (which
+  # is always the case in a Rails process).
+  module ActiveSupportNotifications
+    EVENT_NAME = "sql.pinot"
+
+    def self.install!
+      return if installed?
+
+      Pinot::Instrumentation.on_query = method(:notify)
+      @installed = true
+    end
+
+    def self.installed?
+      @installed || false
+    end
+
+    def self.uninstall!
+      Pinot::Instrumentation.on_query = nil
+      @installed = false
+    end
+
+    def self.notify(event)
+      payload = {
+        sql:      event[:query],
+        name:     event[:table],
+        duration: event[:duration_ms],
+        success:  event[:success]
+      }
+
+      if (err = event[:error])
+        payload[:exception]        = [err.class.name, err.message]
+        payload[:exception_object] = err
+      end
+
+      ::ActiveSupport::Notifications.instrument(EVENT_NAME, payload)
+    end
+    private_class_method :notify
+  end
+end

data/lib/pinot/circuit_breaker.rb

@@ -1,8 +1,34 @@
 module Pinot
-  # Per-broker circuit breaker. States: CLOSED (normal), OPEN (rejecting), HALF_OPEN (probing).
+  # Per-broker circuit breaker implementing the classic three-state machine:
   #
-  # failure_threshold  - consecutive failures before opening (default 5)
-  # open_timeout       - seconds to stay OPEN before moving to HALF_OPEN (default 30)
+  #   CLOSED    — normal operation; failures are counted
+  #   OPEN      — all calls rejected immediately with BrokerCircuitOpenError
+  #   HALF_OPEN — one probe call allowed through; success → CLOSED, failure → OPEN
+  #
+  # A breaker opens after +failure_threshold+ consecutive transport-level failures
+  # (BrokerUnavailableError, connection resets, timeouts). It automatically
+  # transitions to HALF_OPEN after +open_timeout+ seconds.
+  #
+  # Use CircuitBreakerRegistry to share breakers across Connection instances.
+  #
+  # == Configuration
+  #
+  #   config = Pinot::ClientConfig.new(
+  #     broker_list:               ["broker:8099"],
+  #     circuit_breaker_enabled:   true,
+  #     circuit_breaker_threshold: 3,   # open after 3 failures (default 5)
+  #     circuit_breaker_timeout:   10   # reopen probe after 10 s (default 30)
+  #   )
+  #   conn = Pinot.from_config(config)
+  #
+  # == Error class
+  #
+  #   Pinot::CircuitBreaker::BrokerCircuitOpenError
+  #     — raised when the circuit is OPEN; inherits from BrokerNotFoundError
+  #       so callers that already rescue BrokerNotFoundError get it for free.
+  #
+  # @param failure_threshold [Integer] consecutive failures before opening (default 5)
+  # @param open_timeout      [Integer] seconds to wait before probing again (default 30)
   class CircuitBreaker
     CLOSED    = :closed
     OPEN      = :open
@@ -85,7 +111,9 @@ module Pinot
     end
   end
 
-  # Registry of per-broker CircuitBreakers, shared across transport calls.
+  # Thread-safe registry that lazily creates and caches one CircuitBreaker per
+  # broker address string. Shared by all Connection instances built from the
+  # same ClientConfig so that failures from parallel queries accumulate correctly.
   class CircuitBreakerRegistry
     def initialize(failure_threshold: 5, open_timeout: 30)
       @failure_threshold = failure_threshold

data/lib/pinot/connection.rb

@@ -1,7 +1,26 @@
 require "bigdecimal"
 
 module Pinot
+  # Main entry point for querying Apache Pinot over HTTP.
+  #
+  # Build a Connection via the factory helpers rather than instantiating directly:
+  #
+  #   # Static broker list
+  #   conn = Pinot.from_broker_list(["broker1:8099", "broker2:8099"])
+  #
+  #   # Controller-managed broker discovery
+  #   conn = Pinot.from_controller("controller:9000")
+  #
+  #   # Full configuration
+  #   conn = Pinot.from_config(Pinot::ClientConfig.new(
+  #     broker_list:             ["broker:8099"],
+  #     query_timeout_ms:        5_000,
+  #     use_multistage_engine:   true,
+  #     max_retries:             2,
+  #     circuit_breaker_enabled: true
+  #   ))
   class Connection
+    # @return [Integer, nil] default query timeout in milliseconds; can be overridden per-call
     attr_accessor :query_timeout_ms
 
     def initialize(transport:, broker_selector:, use_multistage_engine: false, logger: nil,
@@ -27,6 +46,18 @@ module Pinot
       @trace = false
     end
 
+    # Execute a SQL query against +table+ and return a BrokerResponse.
+    #
+    # @param table           [String]  Pinot table name (used for broker selection)
+    # @param query           [String]  SQL query string
+    # @param query_timeout_ms [Integer, nil] per-call timeout override (ms); overrides
+    #                        the connection-level query_timeout_ms
+    # @param headers         [Hash]    extra HTTP headers merged into this request only
+    # @return [BrokerResponse]
+    # @raise [BrokerNotFoundError]     no broker available for the table
+    # @raise [QueryTimeoutError]       query exceeded the timeout
+    # @raise [BrokerUnavailableError]  broker returned 503/504
+    # @raise [TransportError]          other non-200 HTTP response
     def execute_sql(table, query, query_timeout_ms: nil, headers: {})
       Pinot::Instrumentation.instrument(table: table, query: query) do
         logger.debug "Executing SQL on table=#{table}: #{query}"
@@ -39,10 +70,24 @@ module Pinot
       end
     end
 
+    # Convenience wrapper around execute_sql with an explicit timeout.
+    # Equivalent to execute_sql(table, query, query_timeout_ms: timeout_ms).
     def execute_sql_with_timeout(table, query, timeout_ms)
       execute_sql(table, query, query_timeout_ms: timeout_ms)
     end
 
+    # Execute a parameterised query by substituting +params+ into +query_pattern+.
+    # Each +?+ placeholder in the pattern is replaced by the corresponding value
+    # using safe type-aware formatting (strings are quoted and escaped).
+    #
+    # @param table         [String]  Pinot table name
+    # @param query_pattern [String]  SQL template, e.g. "SELECT * FROM t WHERE id = ?"
+    # @param params        [Array]   ordered values; supported types: String, Integer,
+    #                                Float, TrueClass, FalseClass, BigDecimal, Time
+    # @param query_timeout_ms [Integer, nil] per-call timeout override (ms)
+    # @param headers       [Hash]    extra HTTP headers for this request
+    # @return [BrokerResponse]
+    # @raise [RuntimeError] placeholder / param count mismatch or unsupported type
     def execute_sql_with_params(table, query_pattern, params, query_timeout_ms: nil, headers: {})
       query = format_query(query_pattern, params)
       execute_sql(table, query, query_timeout_ms: query_timeout_ms, headers: headers)
@@ -50,6 +95,29 @@ module Pinot
       raise e
     end
 
+    # Execute multiple queries in parallel and return results in the same order.
+    #
+    # Each element of +queries+ must be a Hash with keys +:table+ and +:query+
+    # (Strings or Symbols). An optional +:query_timeout_ms+ key overrides the
+    # per-query timeout.
+    #
+    # Each slot in the returned Array is a QueryResult with either a +response+ or
+    # an +error+ — failures are isolated so one bad query does not raise for the
+    # whole batch.
+    #
+    #   results = conn.execute_many([
+    #     { table: "orders",   query: "SELECT count(*) FROM orders" },
+    #     { table: "products", query: "SELECT count(*) FROM products" }
+    #   ], max_concurrency: 4)
+    #
+    #   results.each do |r|
+    #     puts r.success? ? r.response.result_table.get_long(0, 0) : r.error.message
+    #   end
+    #
+    # @param queries         [Array<Hash>]  query descriptors
+    # @param max_concurrency [Integer, nil] maximum simultaneous in-flight queries;
+    #                        nil means unlimited
+    # @return [Array<QueryResult>]
     def execute_many(queries, max_concurrency: nil)
       return [] if queries.empty?
 
@@ -79,6 +147,19 @@ module Pinot
       results
     end
 
+    # Return a Paginator for cursor-based iteration over large result sets.
+    #
+    # The query must include a LIMIT clause; the broker stores the result set
+    # and returns it in +page_size+ row slices on demand.
+    #
+    #   paginator = conn.paginate("SELECT * FROM myTable LIMIT 50000", page_size: 500)
+    #   paginator.each_row { |row| puts row.map(&:to_s).join(", ") }
+    #
+    # @param query         [String]  SQL query (should include LIMIT)
+    # @param page_size     [Integer] rows per page (default Paginator::DEFAULT_PAGE_SIZE = 1000)
+    # @param table         [String, nil] used only for broker selection; nil picks any broker
+    # @param extra_headers [Hash]    merged into every HTTP request of this cursor session
+    # @return [Paginator]
     def paginate(query, page_size: Paginator::DEFAULT_PAGE_SIZE, table: nil, extra_headers: {})
       broker = @broker_selector.select_broker(table || "")
       Paginator.new(
@@ -90,6 +171,17 @@ module Pinot
       )
     end
 
+    # Create a PreparedStatement from a query template with +?+ placeholders.
+    #
+    #   stmt = conn.prepare("myTable", "SELECT * FROM myTable WHERE id = ? AND name = ?")
+    #   stmt.set(1, 42)
+    #   stmt.set(2, "Alice")
+    #   resp = stmt.execute
+    #
+    # @param table          [String] Pinot table name (non-empty)
+    # @param query_template [String] SQL with one or more +?+ placeholders
+    # @return [PreparedStatementImpl]
+    # @raise [ArgumentError] if table or query_template is blank, or contains no placeholders
     def prepare(table, query_template)
       raise ArgumentError, "table name cannot be empty" if table.nil? || table.strip.empty?
       raise ArgumentError, "query template cannot be empty" if query_template.nil? || query_template.strip.empty?

data/lib/pinot/connection_factory.rb

@@ -1,9 +1,24 @@
 module Pinot
+  # Build a Connection from a static list of broker addresses.
+  #
+  #   conn = Pinot.from_broker_list(["broker1:8099", "broker2:8099"])
+  #
+  # @param broker_list [Array<String>] broker host:port entries
+  # @param http_client [HttpClient, nil] optional pre-configured HTTP client
+  # @return [Connection]
   def self.from_broker_list(broker_list, http_client: nil)
     config = ClientConfig.new(broker_list: broker_list)
     from_config(config, http_client: http_client)
   end
 
+  # Build a Connection backed by a Pinot controller for automatic broker discovery.
+  # The controller is polled in the background to keep the broker list fresh.
+  #
+  #   conn = Pinot.from_controller("controller:9000")
+  #
+  # @param controller_address [String] controller host:port (or http://host:port)
+  # @param http_client [HttpClient, nil] optional pre-configured HTTP client
+  # @return [Connection]
   def self.from_controller(controller_address, http_client: nil)
     config = ClientConfig.new(
       controller_config: ControllerConfig.new(controller_address: controller_address)
@@ -11,6 +26,25 @@ module Pinot
     from_config(config, http_client: http_client)
   end
 
+  # Build a Connection from a fully specified ClientConfig.
+  # This is the most flexible factory: it handles all transport types (HTTP,
+  # gRPC, ZooKeeper) and wires up the circuit breaker and retry logic from
+  # config flags.
+  #
+  #   config = Pinot::ClientConfig.new(
+  #     broker_list:             ["broker:8099"],
+  #     query_timeout_ms:        5_000,
+  #     use_multistage_engine:   true,
+  #     max_retries:             2,
+  #     retry_interval_ms:       100,
+  #     circuit_breaker_enabled: true
+  #   )
+  #   conn = Pinot.from_config(config)
+  #
+  # @param config     [ClientConfig] fully populated config object
+  # @param http_client [HttpClient, nil] optional pre-configured HTTP client
+  # @return [Connection]
+  # @raise [ConfigurationError] if no broker source is specified in config
   def self.from_config(config, http_client: nil)
     config.validate!
 

data/lib/pinot/instrumentation.rb

@@ -1,14 +1,23 @@
 module Pinot
+  # Low-level instrumentation hook that fires after every query executed via
+  # Connection#execute_sql. This is the extension point used by
+  # Pinot::ActiveSupportNotifications and any custom observability layer.
+  #
+  # Register a single callback:
+  #
+  #   Pinot::Instrumentation.on_query = ->(event) do
+  #     MyMetrics.record(event[:table], event[:duration_ms], event[:success])
+  #   end
+  #
+  # The event Hash contains:
+  #   :table        => String  — table name passed to execute_sql
+  #   :query        => String  — SQL string
+  #   :duration_ms  => Float   — wall-clock time in milliseconds
+  #   :success      => Boolean — false when an exception was raised
+  #   :error        => Exception or nil — the exception on failure, nil on success
+  #
+  # Only one callback can be registered at a time. Set on_query= to nil to remove it.
   module Instrumentation
-    # Called around every query execution.
-    # Implement by setting Pinot::Instrumentation.on_query = proc { |event| ... }
-    # event is a Hash:
-    #   :table        => String
-    #   :query        => String
-    #   :duration_ms  => Float
-    #   :success      => Boolean
-    #   :error        => Exception or nil
-
     def self.on_query=(callback)
       @on_query = callback
     end

data/lib/pinot/paginator.rb

@@ -1,15 +1,43 @@
 module Pinot
-  # Implements Pinot's server-side cursor API.
+  # Cursor-based pagination using Pinot's native server-side cursor API.
   #
   # The broker stores the full result set and returns slices on demand.
-  # All fetch requests after the first must go to the same broker that
-  # owns the cursor state (brokerHost:brokerPort from the initial response).
+  # All fetch requests after the first are pinned to the broker that owns
+  # the cursor (brokerHost:brokerPort from the initial response), ensuring
+  # correct broker affinity.
   #
-  # Usage:
-  #   paginator = conn.paginate("SELECT * FROM t LIMIT 10000", page_size: 100)
+  # == Obtaining a Paginator
+  #
+  #   paginator = conn.paginate(
+  #     "SELECT * FROM myTable WHERE col > 0",
+  #     page_size:     500,    # rows per page (default 1000)
+  #     table:         nil,    # used only for broker selection; omit for single-broker setups
+  #     extra_headers: {}      # merged into every HTTP request
+  #   )
+  #
+  # == Iteration
+  #
+  #   # Page-by-page (each page is a BrokerResponse):
   #   paginator.each_page { |resp| process(resp.result_table) }
-  #   paginator.each_row  { |row|  puts row.map(&:to_s).join(", ") }
-  #   paginator.delete    # optional early cleanup; also called automatically on exhaustion
+  #
+  #   # Row-by-row (each row is an Array of JsonNumber/String cells):
+  #   paginator.each_row { |row| puts row.map(&:to_s).join(", ") }
+  #
+  #   # Enumerable methods work because #each is aliased to #each_row:
+  #   rows = paginator.to_a
+  #   paginator.select { |row| row.first.to_s.to_i > 100 }
+  #
+  # == Cursor lifecycle
+  #
+  # The cursor is deleted from the broker automatically after the last page is
+  # consumed. Call #delete explicitly for early cleanup (e.g. break out of loop).
+  # DELETE failures are swallowed — cursors expire naturally on the broker side.
+  #
+  # == Protocol
+  #
+  # 1. POST /query/sql?getCursor=true&numRows=N  — submit query, get first page + requestId
+  # 2. GET  /responseStore/{id}/results?offset=K&numRows=N  — fetch subsequent pages
+  # 3. DELETE /responseStore/{id}  — release cursor (best-effort)
   class Paginator
     include Enumerable
 

data/lib/pinot/version.rb

@@ -1,3 +1,3 @@
 module Pinot
-  VERSION = "1.26.0"
+  VERSION = "1.27.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pinot-client
 version: !ruby/object:Gem::Version
-  version: 1.26.0
+  version: 1.27.0
 platform: ruby
 authors:
 - Xiang Fu
@@ -103,6 +103,7 @@ files:
 - LICENSE
 - README.md
 - lib/pinot.rb
+- lib/pinot/active_support_notifications.rb
 - lib/pinot/broker_selector.rb
 - lib/pinot/circuit_breaker.rb
 - lib/pinot/config.rb