pinot-client 1.27.0 → 1.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/pinot/active_support_notifications.rb +7 -6
- data/lib/pinot/circuit_breaker.rb +5 -4
- data/lib/pinot/config.rb +1 -3
- data/lib/pinot/connection.rb +45 -18
- data/lib/pinot/connection_factory.rb +1 -1
- data/lib/pinot/controller_based_broker_selector.rb +6 -9
- data/lib/pinot/grpc_transport.rb +1 -1
- data/lib/pinot/instrumentation.rb +69 -13
- data/lib/pinot/open_telemetry.rb +161 -0
- data/lib/pinot/paginator.rb +3 -1
- data/lib/pinot/prepared_statement.rb +11 -12
- data/lib/pinot/proto/broker_service_pb.rb +3 -3
- data/lib/pinot/proto/broker_service_services_pb.rb +3 -4
- data/lib/pinot/response.rb +22 -8
- data/lib/pinot/simple_broker_selector.rb +1 -0
- data/lib/pinot/table_aware_broker_selector.rb +4 -2
- data/lib/pinot/transport.rb +42 -20
- data/lib/pinot/version.rb +1 -1
- data/lib/pinot/zookeeper_broker_selector.rb +11 -5
- data/lib/pinot.rb +1 -1
- metadata +31 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f728a6fe054a729893524d1da24e6ab2cddabda2d17c3bfae183e77271a70b1c
|
|
4
|
+
data.tar.gz: 6f10e6f81f725d9e34ed0d68a3ed04070244308f949a7a01533f90940fff18a9
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 26d0a240f8d335f1acf02363ab974bf8e0323c3ddc045b2c9539340468231d68c6fe6c1c8517e0fb8a79ad4b154bd8178847e890d8475280200b33f327fcb467
|
|
7
|
+
data.tar.gz: 987c72674f5902f79365ff493ff541299e0efcc13374016bee1c52b98018734ff50498d7991c08807f50c5e0b70aff2be734eaa564ba7aa640da6530e66b4665
|
|
@@ -43,12 +43,12 @@ module Pinot
|
|
|
43
43
|
# ActiveSupport::Notifications to already be defined at install! time (which
|
|
44
44
|
# is always the case in a Rails process).
|
|
45
45
|
module ActiveSupportNotifications
|
|
46
|
-
EVENT_NAME = "sql.pinot"
|
|
46
|
+
EVENT_NAME = "sql.pinot".freeze
|
|
47
47
|
|
|
48
48
|
def self.install!
|
|
49
49
|
return if installed?
|
|
50
50
|
|
|
51
|
-
Pinot::Instrumentation.
|
|
51
|
+
@listener = Pinot::Instrumentation.subscribe(method(:notify))
|
|
52
52
|
@installed = true
|
|
53
53
|
end
|
|
54
54
|
|
|
@@ -57,16 +57,17 @@ module Pinot
|
|
|
57
57
|
end
|
|
58
58
|
|
|
59
59
|
def self.uninstall!
|
|
60
|
-
Pinot::Instrumentation.
|
|
60
|
+
Pinot::Instrumentation.unsubscribe(@listener) if @listener
|
|
61
|
+
@listener = nil
|
|
61
62
|
@installed = false
|
|
62
63
|
end
|
|
63
64
|
|
|
64
65
|
def self.notify(event)
|
|
65
66
|
payload = {
|
|
66
|
-
sql:
|
|
67
|
-
name:
|
|
67
|
+
sql: event[:query],
|
|
68
|
+
name: event[:table],
|
|
68
69
|
duration: event[:duration_ms],
|
|
69
|
-
success:
|
|
70
|
+
success: event[:success]
|
|
70
71
|
}
|
|
71
72
|
|
|
72
73
|
if (err = event[:error])
|
|
@@ -34,7 +34,8 @@ module Pinot
|
|
|
34
34
|
OPEN = :open
|
|
35
35
|
HALF_OPEN = :half_open
|
|
36
36
|
|
|
37
|
-
BrokerCircuitOpenError
|
|
37
|
+
class BrokerCircuitOpenError < BrokerNotFoundError
|
|
38
|
+
end
|
|
38
39
|
|
|
39
40
|
attr_reader :state, :failure_count
|
|
40
41
|
|
|
@@ -48,14 +49,14 @@ module Pinot
|
|
|
48
49
|
end
|
|
49
50
|
|
|
50
51
|
# Call the block; record success/failure and enforce open-circuit rejection.
|
|
51
|
-
def call(
|
|
52
|
+
def call(_broker_address)
|
|
52
53
|
@mutex.synchronize { check_state! }
|
|
53
54
|
begin
|
|
54
55
|
result = yield
|
|
55
56
|
@mutex.synchronize { on_success }
|
|
56
57
|
result
|
|
57
58
|
rescue BrokerUnavailableError, Errno::ECONNRESET, Errno::ECONNREFUSED,
|
|
58
|
-
Errno::ETIMEDOUT, Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout
|
|
59
|
+
Errno::ETIMEDOUT, Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout
|
|
59
60
|
@mutex.synchronize { on_failure }
|
|
60
61
|
raise
|
|
61
62
|
end
|
|
@@ -126,7 +127,7 @@ module Pinot
|
|
|
126
127
|
@mutex.synchronize do
|
|
127
128
|
@breakers[broker_address] ||= CircuitBreaker.new(
|
|
128
129
|
failure_threshold: @failure_threshold,
|
|
129
|
-
open_timeout:
|
|
130
|
+
open_timeout: @open_timeout
|
|
130
131
|
)
|
|
131
132
|
end
|
|
132
133
|
end
|
data/lib/pinot/config.rb
CHANGED
|
@@ -89,9 +89,7 @@ module Pinot
|
|
|
89
89
|
raise ConfigurationError, "query_timeout_ms must be positive, got: #{query_timeout_ms}"
|
|
90
90
|
end
|
|
91
91
|
|
|
92
|
-
if !pool_size.nil? && pool_size < 1
|
|
93
|
-
raise ConfigurationError, "pool_size must be at least 1, got: #{pool_size}"
|
|
94
|
-
end
|
|
92
|
+
raise ConfigurationError, "pool_size must be at least 1, got: #{pool_size}" if !pool_size.nil? && pool_size < 1
|
|
95
93
|
|
|
96
94
|
if !keep_alive_timeout.nil? && keep_alive_timeout <= 0
|
|
97
95
|
raise ConfigurationError, "keep_alive_timeout must be positive, got: #{keep_alive_timeout}"
|
data/lib/pinot/connection.rb
CHANGED
|
@@ -34,9 +34,7 @@ module Pinot
|
|
|
34
34
|
@circuit_breaker_registry = circuit_breaker_registry
|
|
35
35
|
end
|
|
36
36
|
|
|
37
|
-
|
|
38
|
-
@use_multistage_engine = val
|
|
39
|
-
end
|
|
37
|
+
attr_writer :use_multistage_engine
|
|
40
38
|
|
|
41
39
|
def open_trace
|
|
42
40
|
@trace = true
|
|
@@ -91,8 +89,6 @@ module Pinot
|
|
|
91
89
|
def execute_sql_with_params(table, query_pattern, params, query_timeout_ms: nil, headers: {})
|
|
92
90
|
query = format_query(query_pattern, params)
|
|
93
91
|
execute_sql(table, query, query_timeout_ms: query_timeout_ms, headers: headers)
|
|
94
|
-
rescue => e
|
|
95
|
-
raise e
|
|
96
92
|
end
|
|
97
93
|
|
|
98
94
|
# Execute multiple queries in parallel and return results in the same order.
|
|
@@ -121,7 +117,7 @@ module Pinot
|
|
|
121
117
|
def execute_many(queries, max_concurrency: nil)
|
|
122
118
|
return [] if queries.empty?
|
|
123
119
|
|
|
124
|
-
results
|
|
120
|
+
results = Array.new(queries.size)
|
|
125
121
|
# Queue acts as a counting semaphore: pre-filled with N tokens.
|
|
126
122
|
sem = max_concurrency ? build_semaphore(max_concurrency) : nil
|
|
127
123
|
|
|
@@ -131,14 +127,14 @@ module Pinot
|
|
|
131
127
|
timeout_ms = item[:query_timeout_ms] || item["query_timeout_ms"]
|
|
132
128
|
|
|
133
129
|
Thread.new do
|
|
134
|
-
sem&.pop
|
|
130
|
+
sem&.pop # acquire
|
|
135
131
|
begin
|
|
136
132
|
resp = execute_sql(table, query, query_timeout_ms: timeout_ms)
|
|
137
133
|
results[idx] = QueryResult.new(table: table, query: query, response: resp, error: nil)
|
|
138
|
-
rescue => e
|
|
134
|
+
rescue StandardError => e
|
|
139
135
|
results[idx] = QueryResult.new(table: table, query: query, response: nil, error: e)
|
|
140
136
|
ensure
|
|
141
|
-
sem&.push(:token)
|
|
137
|
+
sem&.push(:token) # release
|
|
142
138
|
end
|
|
143
139
|
end
|
|
144
140
|
end
|
|
@@ -166,11 +162,42 @@ module Pinot
|
|
|
166
162
|
@transport.http_client,
|
|
167
163
|
broker,
|
|
168
164
|
query,
|
|
169
|
-
page_size:
|
|
165
|
+
page_size: page_size,
|
|
170
166
|
extra_headers: extra_headers
|
|
171
167
|
)
|
|
172
168
|
end
|
|
173
169
|
|
|
170
|
+
# Check whether a broker is reachable and responding to queries.
|
|
171
|
+
#
|
|
172
|
+
# Runs a lightweight broker-side liveness check: first tries the dedicated
|
|
173
|
+
# +/health+ HTTP endpoint (returns 200 when the broker is healthy), and falls
|
|
174
|
+
# back to executing "SELECT 1 FROM DUAL" if the endpoint is unavailable.
|
|
175
|
+
#
|
|
176
|
+
# Returns +true+ when the broker responds successfully, +false+ on any
|
|
177
|
+
# error (connection refused, timeout, non-200 response, etc.). Never raises.
|
|
178
|
+
#
|
|
179
|
+
# Intended for Kubernetes readiness / liveness probes and health-check
|
|
180
|
+
# endpoints in Rails / Rack applications:
|
|
181
|
+
#
|
|
182
|
+
# get "/healthz" do
|
|
183
|
+
# conn.healthy? ? [200, "OK"] : [503, "Pinot unavailable"]
|
|
184
|
+
# end
|
|
185
|
+
#
|
|
186
|
+
# @param table [String, nil] table used for broker selection (nil = any broker)
|
|
187
|
+
# @param timeout_ms [Integer] per-check timeout in ms (default 2000)
|
|
188
|
+
# @return [Boolean]
|
|
189
|
+
def healthy?(table: nil, timeout_ms: 2_000)
|
|
190
|
+
broker = @broker_selector.select_broker(table || "")
|
|
191
|
+
base = broker.start_with?("http://", "https://") ? broker : "http://#{broker}"
|
|
192
|
+
client = HttpClient.new(timeout: timeout_ms / 1000.0)
|
|
193
|
+
resp = client.get("#{base}/health", headers: {})
|
|
194
|
+
resp.code.to_i == 200
|
|
195
|
+
rescue StandardError
|
|
196
|
+
false
|
|
197
|
+
ensure
|
|
198
|
+
client&.close
|
|
199
|
+
end
|
|
200
|
+
|
|
174
201
|
# Create a PreparedStatement from a query template with +?+ placeholders.
|
|
175
202
|
#
|
|
176
203
|
# stmt = conn.prepare("myTable", "SELECT * FROM myTable WHERE id = ? AND name = ?")
|
|
@@ -185,8 +212,10 @@ module Pinot
|
|
|
185
212
|
def prepare(table, query_template)
|
|
186
213
|
raise ArgumentError, "table name cannot be empty" if table.nil? || table.strip.empty?
|
|
187
214
|
raise ArgumentError, "query template cannot be empty" if query_template.nil? || query_template.strip.empty?
|
|
215
|
+
|
|
188
216
|
count = query_template.count("?")
|
|
189
217
|
raise ArgumentError, "query template must contain at least one parameter placeholder (?)" if count == 0
|
|
218
|
+
|
|
190
219
|
PreparedStatementImpl.new(connection: self, table: table, query_template: query_template)
|
|
191
220
|
end
|
|
192
221
|
|
|
@@ -196,12 +225,13 @@ module Pinot
|
|
|
196
225
|
if placeholders != params.length
|
|
197
226
|
raise "failed to format query: number of placeholders in queryPattern (#{placeholders}) does not match number of params (#{params.length})"
|
|
198
227
|
end
|
|
228
|
+
|
|
199
229
|
parts = pattern.split("?", -1)
|
|
200
230
|
result = ""
|
|
201
231
|
params.each_with_index do |param, i|
|
|
202
232
|
formatted = begin
|
|
203
233
|
format_arg(param)
|
|
204
|
-
rescue => e
|
|
234
|
+
rescue StandardError => e
|
|
205
235
|
raise "failed to format query: failed to format parameter: #{e.message}"
|
|
206
236
|
end
|
|
207
237
|
result += parts[i] + formatted
|
|
@@ -213,11 +243,7 @@ module Pinot
|
|
|
213
243
|
case value
|
|
214
244
|
when String
|
|
215
245
|
"'#{value.gsub("'", "''")}'"
|
|
216
|
-
when Integer
|
|
217
|
-
value.to_s
|
|
218
|
-
when Float
|
|
219
|
-
value.to_s
|
|
220
|
-
when TrueClass, FalseClass
|
|
246
|
+
when Integer, Float, TrueClass, FalseClass
|
|
221
247
|
value.to_s
|
|
222
248
|
when BigDecimal
|
|
223
249
|
s = value.to_s("F")
|
|
@@ -239,9 +265,10 @@ module Pinot
|
|
|
239
265
|
q
|
|
240
266
|
end
|
|
241
267
|
|
|
242
|
-
def run_with_circuit_breaker(broker, &
|
|
268
|
+
def run_with_circuit_breaker(broker, &)
|
|
243
269
|
return yield unless @circuit_breaker_registry
|
|
244
|
-
|
|
270
|
+
|
|
271
|
+
@circuit_breaker_registry.for(broker).call(broker, &)
|
|
245
272
|
end
|
|
246
273
|
|
|
247
274
|
def logger
|
|
@@ -138,7 +138,7 @@ module Pinot
|
|
|
138
138
|
|
|
139
139
|
CircuitBreakerRegistry.new(
|
|
140
140
|
failure_threshold: config.circuit_breaker_threshold || 5,
|
|
141
|
-
open_timeout:
|
|
141
|
+
open_timeout: config.circuit_breaker_timeout || 30
|
|
142
142
|
)
|
|
143
143
|
end
|
|
144
144
|
private_class_method :build_circuit_breaker_registry
|
|
@@ -4,7 +4,7 @@ require "json"
|
|
|
4
4
|
|
|
5
5
|
module Pinot
|
|
6
6
|
class ControllerBasedBrokerSelector < TableAwareBrokerSelector
|
|
7
|
-
CONTROLLER_API_PATH = "/v2/brokers/tables?state=ONLINE"
|
|
7
|
+
CONTROLLER_API_PATH = "/v2/brokers/tables?state=ONLINE".freeze
|
|
8
8
|
DEFAULT_UPDATE_FREQ_MS = 1000
|
|
9
9
|
|
|
10
10
|
def initialize(config, http_client = nil, logger: nil)
|
|
@@ -27,9 +27,8 @@ module Pinot
|
|
|
27
27
|
addr = address.to_s
|
|
28
28
|
if addr.include?("://")
|
|
29
29
|
scheme = addr.split("://").first
|
|
30
|
-
unless %w[http https].include?(scheme)
|
|
31
|
-
|
|
32
|
-
end
|
|
30
|
+
raise ConfigurationError, "unsupported controller URL scheme: #{scheme}" unless %w[http https].include?(scheme)
|
|
31
|
+
|
|
33
32
|
addr.chomp("/") + CONTROLLER_API_PATH
|
|
34
33
|
else
|
|
35
34
|
"http://#{addr.chomp("/")}#{CONTROLLER_API_PATH}"
|
|
@@ -40,13 +39,11 @@ module Pinot
|
|
|
40
39
|
|
|
41
40
|
def fetch_and_update
|
|
42
41
|
headers = { "Accept" => "application/json" }
|
|
43
|
-
|
|
42
|
+
.merge(@config.extra_controller_api_headers || {})
|
|
44
43
|
|
|
45
44
|
resp = @internal_http.get(@controller_url, headers: headers)
|
|
46
45
|
|
|
47
|
-
unless resp.code.to_i == 200
|
|
48
|
-
raise TransportError, "controller API returned HTTP status code #{resp.code}"
|
|
49
|
-
end
|
|
46
|
+
raise TransportError, "controller API returned HTTP status code #{resp.code}" unless resp.code.to_i == 200
|
|
50
47
|
|
|
51
48
|
body = resp.body
|
|
52
49
|
begin
|
|
@@ -70,7 +67,7 @@ module Pinot
|
|
|
70
67
|
sleep interval
|
|
71
68
|
begin
|
|
72
69
|
fetch_and_update
|
|
73
|
-
rescue => e
|
|
70
|
+
rescue StandardError => e
|
|
74
71
|
logger.warn "Pinot controller refresh failed: #{e.message}"
|
|
75
72
|
end
|
|
76
73
|
end
|
data/lib/pinot/grpc_transport.rb
CHANGED
|
@@ -1,45 +1,101 @@
|
|
|
1
1
|
module Pinot
|
|
2
2
|
# Low-level instrumentation hook that fires after every query executed via
|
|
3
3
|
# Connection#execute_sql. This is the extension point used by
|
|
4
|
-
# Pinot::ActiveSupportNotifications and any custom
|
|
4
|
+
# Pinot::ActiveSupportNotifications, Pinot::OpenTelemetry, and any custom
|
|
5
|
+
# observability layer.
|
|
5
6
|
#
|
|
6
|
-
#
|
|
7
|
+
# == Subscribing (multiple listeners supported)
|
|
7
8
|
#
|
|
8
|
-
# Pinot::Instrumentation.
|
|
9
|
+
# listener = Pinot::Instrumentation.subscribe(->(event) do
|
|
9
10
|
# MyMetrics.record(event[:table], event[:duration_ms], event[:success])
|
|
11
|
+
# end)
|
|
12
|
+
#
|
|
13
|
+
# # Remove later:
|
|
14
|
+
# Pinot::Instrumentation.unsubscribe(listener)
|
|
15
|
+
#
|
|
16
|
+
# == Legacy single-callback API (still supported)
|
|
17
|
+
#
|
|
18
|
+
# Pinot::Instrumentation.on_query = ->(event) { ... }
|
|
19
|
+
# Pinot::Instrumentation.on_query = nil # remove
|
|
20
|
+
#
|
|
21
|
+
# == Around-execution hook (for OTel and similar span-based tools)
|
|
22
|
+
#
|
|
23
|
+
# The `around` hook wraps the entire query execution — the block yields the
|
|
24
|
+
# query, and the hook is responsible for calling Instrumentation.notify when
|
|
25
|
+
# done. Only one around hook can be registered at a time.
|
|
26
|
+
#
|
|
27
|
+
# Pinot::Instrumentation.around = ->(table:, query:) do
|
|
28
|
+
# MyTracer.in_span("pinot") { yield }
|
|
10
29
|
# end
|
|
11
30
|
#
|
|
12
|
-
#
|
|
31
|
+
# == Event Hash keys
|
|
32
|
+
#
|
|
13
33
|
# :table => String — table name passed to execute_sql
|
|
14
34
|
# :query => String — SQL string
|
|
15
35
|
# :duration_ms => Float — wall-clock time in milliseconds
|
|
16
36
|
# :success => Boolean — false when an exception was raised
|
|
17
37
|
# :error => Exception or nil — the exception on failure, nil on success
|
|
18
|
-
#
|
|
19
|
-
# Only one callback can be registered at a time. Set on_query= to nil to remove it.
|
|
20
38
|
module Instrumentation
|
|
39
|
+
@listeners = []
|
|
40
|
+
@around = nil
|
|
41
|
+
|
|
42
|
+
# Add a post-execution listener. Returns the listener so it can be passed
|
|
43
|
+
# to unsubscribe later.
|
|
44
|
+
def self.subscribe(listener)
|
|
45
|
+
@listeners << listener
|
|
46
|
+
listener
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Remove a previously subscribed listener.
|
|
50
|
+
def self.unsubscribe(listener)
|
|
51
|
+
@listeners.delete(listener)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Register an around-execution wrapper. Only one wrapper is supported;
|
|
55
|
+
# the new value replaces any previous one. Set to nil to remove.
|
|
56
|
+
def self.around=(wrapper)
|
|
57
|
+
@around = wrapper
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def self.around
|
|
61
|
+
@around
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Legacy single-callback setter. Replaces all listeners with the given
|
|
65
|
+
# callback (or clears them when nil).
|
|
21
66
|
def self.on_query=(callback)
|
|
22
|
-
@
|
|
67
|
+
@listeners = callback ? [callback] : []
|
|
23
68
|
end
|
|
24
69
|
|
|
70
|
+
# Returns the first registered listener (legacy compat).
|
|
25
71
|
def self.on_query
|
|
26
|
-
@
|
|
72
|
+
@listeners.first
|
|
27
73
|
end
|
|
28
74
|
|
|
29
75
|
def self.instrument(table:, query:)
|
|
76
|
+
if @around
|
|
77
|
+
@around.call(table: table, query: query) { yield }
|
|
78
|
+
else
|
|
79
|
+
_timed_instrument(table: table, query: query) { yield }
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Fire all registered listeners with an event hash. Called by the default
|
|
84
|
+
# instrument path and by the OTel around wrapper.
|
|
85
|
+
def self.notify(event)
|
|
86
|
+
@listeners.each { |l| l.call(event) }
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
private_class_method def self._timed_instrument(table:, query:)
|
|
30
90
|
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
31
91
|
result = yield
|
|
32
92
|
duration_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000
|
|
33
93
|
notify(table: table, query: query, duration_ms: duration_ms, success: true, error: nil)
|
|
34
94
|
result
|
|
35
|
-
rescue => e
|
|
95
|
+
rescue StandardError => e
|
|
36
96
|
duration_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000
|
|
37
97
|
notify(table: table, query: query, duration_ms: duration_ms, success: false, error: e)
|
|
38
98
|
raise
|
|
39
99
|
end
|
|
40
|
-
|
|
41
|
-
def self.notify(event)
|
|
42
|
-
@on_query&.call(event)
|
|
43
|
-
end
|
|
44
100
|
end
|
|
45
101
|
end
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
require "pinot"
|
|
2
|
+
|
|
3
|
+
module Pinot
|
|
4
|
+
# Opt-in OpenTelemetry bridge.
|
|
5
|
+
#
|
|
6
|
+
# == Setup
|
|
7
|
+
#
|
|
8
|
+
# Add to an initializer after the opentelemetry SDK is configured:
|
|
9
|
+
#
|
|
10
|
+
# require "pinot/open_telemetry"
|
|
11
|
+
# Pinot::OpenTelemetry.install!
|
|
12
|
+
#
|
|
13
|
+
# == What it does
|
|
14
|
+
#
|
|
15
|
+
# Each call to Connection#execute_sql (and anything built on top of it —
|
|
16
|
+
# execute_sql_with_params, execute_many, PreparedStatement) creates an OTel
|
|
17
|
+
# span named "pinot.query" with attributes following the OpenTelemetry
|
|
18
|
+
# semantic conventions for database spans:
|
|
19
|
+
#
|
|
20
|
+
# db.system = "pinot"
|
|
21
|
+
# db.statement = "<sql>" (the full SQL string)
|
|
22
|
+
# db.name = "<table>" (the Pinot table name)
|
|
23
|
+
# db.operation = "SELECT" (first token of the SQL)
|
|
24
|
+
#
|
|
25
|
+
# On failure the span is marked with error status and the exception is
|
|
26
|
+
# recorded on the span.
|
|
27
|
+
#
|
|
28
|
+
# == Trace-context propagation
|
|
29
|
+
#
|
|
30
|
+
# When installed, every outbound HTTP request to a broker is injected with
|
|
31
|
+
# W3C Trace Context headers (traceparent / tracestate) so distributed traces
|
|
32
|
+
# flow through Pinot. This relies on OpenTelemetry.propagation being
|
|
33
|
+
# configured (the default SDK sets this up automatically).
|
|
34
|
+
#
|
|
35
|
+
# == Feature flag
|
|
36
|
+
#
|
|
37
|
+
# The bridge can be toggled at runtime without reinstalling:
|
|
38
|
+
#
|
|
39
|
+
# Pinot::OpenTelemetry.enabled = false # pause tracing (e.g. in tests)
|
|
40
|
+
# Pinot::OpenTelemetry.enabled = true # resume
|
|
41
|
+
# Pinot::OpenTelemetry.enabled? # => true / false
|
|
42
|
+
#
|
|
43
|
+
# == Lifecycle
|
|
44
|
+
#
|
|
45
|
+
# Pinot::OpenTelemetry.install! # idempotent
|
|
46
|
+
# Pinot::OpenTelemetry.installed? # => true
|
|
47
|
+
# Pinot::OpenTelemetry.uninstall! # removes hooks; leaves transport unpatched
|
|
48
|
+
#
|
|
49
|
+
# Note: this gem does NOT depend on opentelemetry-api or opentelemetry-sdk.
|
|
50
|
+
# Both must be present and initialized before install! is called.
|
|
51
|
+
module OpenTelemetry
|
|
52
|
+
SPAN_NAME = "pinot.query".freeze
|
|
53
|
+
DB_SYSTEM = "pinot".freeze
|
|
54
|
+
TRACER_NAME = "pinot-client".freeze
|
|
55
|
+
|
|
56
|
+
@installed = false
|
|
57
|
+
@enabled = true
|
|
58
|
+
|
|
59
|
+
# Enable or disable tracing at runtime without uninstalling.
|
|
60
|
+
def self.enabled=(val)
|
|
61
|
+
@enabled = val ? true : false
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def self.enabled?
|
|
65
|
+
@enabled
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def self.install!
|
|
69
|
+
return if installed?
|
|
70
|
+
|
|
71
|
+
_install_around_hook
|
|
72
|
+
_patch_transport
|
|
73
|
+
@installed = true
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def self.installed?
|
|
77
|
+
@installed
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def self.uninstall!
|
|
81
|
+
::Pinot::Instrumentation.around = nil
|
|
82
|
+
@installed = false
|
|
83
|
+
# NOTE: JsonHttpTransport prepend is permanent once applied (Ruby limitation).
|
|
84
|
+
# Disable the propagator by unsetting the flag — it no-ops when disabled.
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# -------------------------------------------------------------------------
|
|
88
|
+
# Internal helpers
|
|
89
|
+
# -------------------------------------------------------------------------
|
|
90
|
+
|
|
91
|
+
def self._install_around_hook
|
|
92
|
+
::Pinot::Instrumentation.around = method(:_around)
|
|
93
|
+
end
|
|
94
|
+
private_class_method :_install_around_hook
|
|
95
|
+
|
|
96
|
+
def self._around(table:, query:)
|
|
97
|
+
unless @enabled
|
|
98
|
+
# Bypass tracing; still time and notify listeners.
|
|
99
|
+
return ::Pinot::Instrumentation.send(:_timed_instrument, table: table, query: query) { yield }
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
tracer = ::OpenTelemetry.tracer_provider.tracer(TRACER_NAME, ::Pinot::VERSION)
|
|
103
|
+
attrs = _span_attributes(table, query)
|
|
104
|
+
|
|
105
|
+
tracer.in_span(SPAN_NAME, attributes: attrs, kind: :client) do |span|
|
|
106
|
+
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
107
|
+
begin
|
|
108
|
+
result = yield
|
|
109
|
+
duration_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000
|
|
110
|
+
::Pinot::Instrumentation.notify(
|
|
111
|
+
table: table, query: query, duration_ms: duration_ms, success: true, error: nil
|
|
112
|
+
)
|
|
113
|
+
result
|
|
114
|
+
rescue StandardError => e
|
|
115
|
+
duration_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000
|
|
116
|
+
span.record_exception(e)
|
|
117
|
+
span.status = ::OpenTelemetry::Trace::Status.error(e.message)
|
|
118
|
+
::Pinot::Instrumentation.notify(
|
|
119
|
+
table: table, query: query, duration_ms: duration_ms, success: false, error: e
|
|
120
|
+
)
|
|
121
|
+
raise
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
private_class_method :_around
|
|
126
|
+
|
|
127
|
+
def self._span_attributes(table, query)
|
|
128
|
+
attrs = {
|
|
129
|
+
"db.system" => DB_SYSTEM,
|
|
130
|
+
"db.statement" => query,
|
|
131
|
+
"db.name" => table.to_s
|
|
132
|
+
}
|
|
133
|
+
op = query.to_s.lstrip.split(/\s+/, 2).first&.upcase
|
|
134
|
+
attrs["db.operation"] = op if op && !op.empty?
|
|
135
|
+
attrs
|
|
136
|
+
end
|
|
137
|
+
private_class_method :_span_attributes
|
|
138
|
+
|
|
139
|
+
# Patch JsonHttpTransport to inject W3C trace-context headers into every
|
|
140
|
+
# outbound HTTP request. Applied once via Module#prepend.
|
|
141
|
+
def self._patch_transport
|
|
142
|
+
return if ::Pinot::JsonHttpTransport.ancestors.include?(TraceContextInjector)
|
|
143
|
+
|
|
144
|
+
::Pinot::JsonHttpTransport.prepend(TraceContextInjector)
|
|
145
|
+
end
|
|
146
|
+
private_class_method :_patch_transport
|
|
147
|
+
|
|
148
|
+
# Prepended into JsonHttpTransport. Injects traceparent/tracestate into
|
|
149
|
+
# request headers when the bridge is enabled and a current span exists.
|
|
150
|
+
module TraceContextInjector
|
|
151
|
+
def execute(broker_address, request, extra_request_headers: {})
|
|
152
|
+
otel = ::Pinot::OpenTelemetry
|
|
153
|
+
return super unless otel.installed? && otel.enabled?
|
|
154
|
+
|
|
155
|
+
carrier = {}
|
|
156
|
+
::OpenTelemetry.propagation.inject(carrier)
|
|
157
|
+
super(broker_address, request, extra_request_headers: extra_request_headers.merge(carrier))
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
end
|
data/lib/pinot/paginator.rb
CHANGED
|
@@ -53,7 +53,7 @@ module Pinot
|
|
|
53
53
|
@extra_headers = extra_headers
|
|
54
54
|
|
|
55
55
|
@request_id = nil
|
|
56
|
-
@cursor_base = nil
|
|
56
|
+
@cursor_base = nil # "http://host:port" — set after first response
|
|
57
57
|
@exhausted = false
|
|
58
58
|
end
|
|
59
59
|
|
|
@@ -136,11 +136,13 @@ module Pinot
|
|
|
136
136
|
|
|
137
137
|
def broker_base(address)
|
|
138
138
|
return address if address.start_with?("http://", "https://")
|
|
139
|
+
|
|
139
140
|
"http://#{address}"
|
|
140
141
|
end
|
|
141
142
|
|
|
142
143
|
def broker_base_from_response(resp)
|
|
143
144
|
return nil unless resp.broker_host && resp.broker_port
|
|
145
|
+
|
|
144
146
|
"http://#{resp.broker_host}:#{resp.broker_port}"
|
|
145
147
|
end
|
|
146
148
|
|
|
@@ -52,9 +52,8 @@ module Pinot
|
|
|
52
52
|
def set(index, value)
|
|
53
53
|
@mutex.synchronize do
|
|
54
54
|
raise PreparedStatementClosedError, "prepared statement is closed" if @closed
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
end
|
|
55
|
+
raise "parameter index #{index} is out of range [1, #{@param_count}]" unless index.between?(1, @param_count)
|
|
56
|
+
|
|
58
57
|
@parameters[index - 1] = value
|
|
59
58
|
end
|
|
60
59
|
nil
|
|
@@ -63,13 +62,14 @@ module Pinot
|
|
|
63
62
|
def execute(headers: {})
|
|
64
63
|
@mutex.synchronize do
|
|
65
64
|
raise PreparedStatementClosedError, "prepared statement is closed" if @closed
|
|
65
|
+
|
|
66
66
|
@parameters.each_with_index do |p, i|
|
|
67
67
|
raise "parameter at index #{i + 1} is not set" if p.nil?
|
|
68
68
|
end
|
|
69
69
|
end
|
|
70
70
|
query = begin
|
|
71
71
|
build_query(@parameters)
|
|
72
|
-
rescue => e
|
|
72
|
+
rescue StandardError => e
|
|
73
73
|
raise "failed to build query: #{e.message}"
|
|
74
74
|
end
|
|
75
75
|
@connection.execute_sql(@table, query, headers: headers)
|
|
@@ -77,12 +77,11 @@ module Pinot
|
|
|
77
77
|
|
|
78
78
|
def execute_with_params(*params, headers: {})
|
|
79
79
|
@mutex.synchronize { raise PreparedStatementClosedError, "prepared statement is closed" if @closed }
|
|
80
|
-
if params.length != @param_count
|
|
81
|
-
|
|
82
|
-
end
|
|
80
|
+
raise "expected #{@param_count} parameters, got #{params.length}" if params.length != @param_count
|
|
81
|
+
|
|
83
82
|
query = begin
|
|
84
83
|
build_query(params)
|
|
85
|
-
rescue => e
|
|
84
|
+
rescue StandardError => e
|
|
86
85
|
raise "failed to build query: #{e.message}"
|
|
87
86
|
end
|
|
88
87
|
@connection.execute_sql(@table, query, headers: headers)
|
|
@@ -91,6 +90,7 @@ module Pinot
|
|
|
91
90
|
def clear_parameters
|
|
92
91
|
@mutex.synchronize do
|
|
93
92
|
raise PreparedStatementClosedError, "prepared statement is closed" if @closed
|
|
93
|
+
|
|
94
94
|
@parameters.fill(nil)
|
|
95
95
|
end
|
|
96
96
|
nil
|
|
@@ -105,14 +105,13 @@ module Pinot
|
|
|
105
105
|
end
|
|
106
106
|
|
|
107
107
|
def build_query(params)
|
|
108
|
-
if params.length != @param_count
|
|
109
|
-
|
|
110
|
-
end
|
|
108
|
+
raise "expected #{@param_count} parameters, got #{params.length}" if params.length != @param_count
|
|
109
|
+
|
|
111
110
|
result = ""
|
|
112
111
|
params.each_with_index do |param, i|
|
|
113
112
|
formatted = begin
|
|
114
113
|
@connection.format_arg(param)
|
|
115
|
-
rescue => e
|
|
114
|
+
rescue StandardError => e
|
|
116
115
|
raise "failed to format parameter: #{e.message}"
|
|
117
116
|
end
|
|
118
117
|
result += @parts[i] + formatted
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
+
|
|
2
3
|
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
3
4
|
# source: broker_service.proto
|
|
4
5
|
|
|
5
|
-
require
|
|
6
|
-
|
|
6
|
+
require "google/protobuf"
|
|
7
7
|
|
|
8
8
|
descriptor_data = "\n\x14\x62roker_service.proto\x12\x11pinot.broker.grpc\"\x8f\x01\n\rBrokerRequest\x12\x0b\n\x03sql\x18\x01 \x01(\t\x12@\n\x08metadata\x18\x02 \x03(\x0b\x32..pinot.broker.grpc.BrokerRequest.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\":\n\x0e\x42rokerResponse\x12\x17\n\x0fresult_row_size\x18\x01 \x01(\x05\x12\x0f\n\x07payload\x18\x02 \x01(\x0c\x32m\n\x1cPinotClientGrpcBrokerService\x12M\n\x06Submit\x12 .pinot.broker.grpc.BrokerRequest\x1a!.pinot.broker.grpc.BrokerResponseb\x06proto3"
|
|
9
9
|
|
|
10
|
-
pool =
|
|
10
|
+
pool = Google::Protobuf::DescriptorPool.generated_pool
|
|
11
11
|
pool.add_serialized_file(descriptor_data)
|
|
12
12
|
|
|
13
13
|
module Pinot
|
|
@@ -1,20 +1,19 @@
|
|
|
1
1
|
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
2
2
|
# Source: broker_service.proto for package 'pinot.broker.grpc'
|
|
3
3
|
|
|
4
|
-
require
|
|
5
|
-
require_relative
|
|
4
|
+
require "grpc"
|
|
5
|
+
require_relative "broker_service_pb"
|
|
6
6
|
|
|
7
7
|
module Pinot
|
|
8
8
|
module Broker
|
|
9
9
|
module Grpc
|
|
10
10
|
module PinotClientGrpcBrokerService
|
|
11
11
|
class Service
|
|
12
|
-
|
|
13
12
|
include ::GRPC::GenericService
|
|
14
13
|
|
|
15
14
|
self.marshal_class_method = :encode
|
|
16
15
|
self.unmarshal_class_method = :decode
|
|
17
|
-
self.service_name =
|
|
16
|
+
self.service_name = "pinot.broker.grpc.PinotClientGrpcBrokerService"
|
|
18
17
|
|
|
19
18
|
rpc :Submit, ::Pinot::Broker::Grpc::BrokerRequest, ::Pinot::Broker::Grpc::BrokerResponse
|
|
20
19
|
end
|
data/lib/pinot/response.rb
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
require "bigdecimal"
|
|
2
2
|
|
|
3
3
|
module Pinot
|
|
4
|
-
INT32_MAX =
|
|
4
|
+
INT32_MAX = 2_147_483_647
|
|
5
5
|
INT32_MIN = -2_147_483_648
|
|
6
|
-
INT64_MAX =
|
|
6
|
+
INT64_MAX = 9_223_372_036_854_775_807
|
|
7
7
|
INT64_MIN = -9_223_372_036_854_775_808
|
|
8
8
|
FLOAT32_MAX = 3.4028235e+38
|
|
9
9
|
|
|
@@ -110,14 +110,20 @@ module Pinot
|
|
|
110
110
|
if raw.include?(".") || raw.include?("e") || raw.include?("E")
|
|
111
111
|
# Floating point string — check if it's a whole number
|
|
112
112
|
bd = BigDecimal(raw)
|
|
113
|
-
|
|
113
|
+
begin
|
|
114
|
+
return 0 if bd.infinite? || bd.nan?
|
|
115
|
+
rescue StandardError
|
|
116
|
+
return 0
|
|
117
|
+
end
|
|
114
118
|
int_val = bd.to_i
|
|
115
119
|
return 0 unless bd == BigDecimal(int_val.to_s)
|
|
116
120
|
return 0 if int_val > INT32_MAX || int_val < INT32_MIN
|
|
121
|
+
|
|
117
122
|
int_val.to_i
|
|
118
123
|
else
|
|
119
124
|
int_val = Integer(raw)
|
|
120
125
|
return 0 if int_val > INT32_MAX || int_val < INT32_MIN
|
|
126
|
+
|
|
121
127
|
int_val
|
|
122
128
|
end
|
|
123
129
|
rescue ArgumentError, TypeError
|
|
@@ -133,16 +139,21 @@ module Pinot
|
|
|
133
139
|
begin
|
|
134
140
|
if raw.include?(".") || raw.include?("e") || raw.include?("E")
|
|
135
141
|
bd = BigDecimal(raw)
|
|
136
|
-
|
|
142
|
+
begin
|
|
143
|
+
return 0 if bd.infinite? || bd.nan?
|
|
144
|
+
rescue StandardError
|
|
145
|
+
return 0
|
|
146
|
+
end
|
|
137
147
|
int_val = bd.to_i
|
|
138
148
|
return 0 unless bd == BigDecimal(int_val.to_s)
|
|
139
|
-
|
|
140
|
-
int_val
|
|
149
|
+
|
|
141
150
|
else
|
|
142
151
|
int_val = Integer(raw)
|
|
143
|
-
|
|
144
|
-
int_val
|
|
152
|
+
|
|
145
153
|
end
|
|
154
|
+
return 0 if int_val > INT64_MAX || int_val < INT64_MIN
|
|
155
|
+
|
|
156
|
+
int_val
|
|
146
157
|
rescue ArgumentError, TypeError
|
|
147
158
|
0
|
|
148
159
|
end
|
|
@@ -156,8 +167,10 @@ module Pinot
|
|
|
156
167
|
begin
|
|
157
168
|
f = Float(raw)
|
|
158
169
|
return 0.0 if f.infinite? || f.nan?
|
|
170
|
+
|
|
159
171
|
f32 = f.to_f
|
|
160
172
|
return 0.0 if f32.abs > FLOAT32_MAX
|
|
173
|
+
|
|
161
174
|
f32
|
|
162
175
|
rescue ArgumentError, TypeError
|
|
163
176
|
0.0
|
|
@@ -172,6 +185,7 @@ module Pinot
|
|
|
172
185
|
begin
|
|
173
186
|
f = Float(raw)
|
|
174
187
|
return 0.0 if f.infinite? || f.nan?
|
|
188
|
+
|
|
175
189
|
f
|
|
176
190
|
rescue ArgumentError, TypeError
|
|
177
191
|
0.0
|
|
@@ -2,8 +2,8 @@ module Pinot
|
|
|
2
2
|
class TableAwareBrokerSelector
|
|
3
3
|
include BrokerSelector
|
|
4
4
|
|
|
5
|
-
OFFLINE_SUFFIX = "_OFFLINE"
|
|
6
|
-
REALTIME_SUFFIX = "_REALTIME"
|
|
5
|
+
OFFLINE_SUFFIX = "_OFFLINE".freeze
|
|
6
|
+
REALTIME_SUFFIX = "_REALTIME".freeze
|
|
7
7
|
|
|
8
8
|
def initialize
|
|
9
9
|
@mutex = Mutex.new
|
|
@@ -20,11 +20,13 @@ module Pinot
|
|
|
20
20
|
@mutex.synchronize do
|
|
21
21
|
if table_name.empty?
|
|
22
22
|
raise BrokerNotFoundError, "no available broker" if @all_broker_list.empty?
|
|
23
|
+
|
|
23
24
|
return @all_broker_list.sample
|
|
24
25
|
end
|
|
25
26
|
brokers = @table_broker_map[table_name]
|
|
26
27
|
raise TableNotFoundError, "unable to find table: #{table}" unless brokers
|
|
27
28
|
raise BrokerNotFoundError, "no available broker for table: #{table}" if brokers.empty?
|
|
29
|
+
|
|
28
30
|
brokers.sample
|
|
29
31
|
end
|
|
30
32
|
end
|
data/lib/pinot/transport.rb
CHANGED
|
@@ -50,10 +50,18 @@ module Pinot
|
|
|
50
50
|
end
|
|
51
51
|
|
|
52
52
|
def close
|
|
53
|
-
|
|
53
|
+
begin
|
|
54
|
+
@reaper.kill
|
|
55
|
+
rescue StandardError
|
|
56
|
+
nil
|
|
57
|
+
end
|
|
54
58
|
@pool_mutex.synchronize do
|
|
55
59
|
@pool.each_value do |entries|
|
|
56
|
-
entries.each
|
|
60
|
+
entries.each do |entry|
|
|
61
|
+
entry.http.finish
|
|
62
|
+
rescue StandardError
|
|
63
|
+
nil
|
|
64
|
+
end
|
|
57
65
|
end
|
|
58
66
|
@pool.clear
|
|
59
67
|
end
|
|
@@ -69,8 +77,12 @@ module Pinot
|
|
|
69
77
|
result = yield http
|
|
70
78
|
checkin(key, http)
|
|
71
79
|
result
|
|
72
|
-
rescue => e
|
|
73
|
-
|
|
80
|
+
rescue StandardError => e
|
|
81
|
+
begin
|
|
82
|
+
http.finish
|
|
83
|
+
rescue StandardError
|
|
84
|
+
nil
|
|
85
|
+
end
|
|
74
86
|
raise e
|
|
75
87
|
end
|
|
76
88
|
end
|
|
@@ -85,7 +97,11 @@ module Pinot
|
|
|
85
97
|
fresh = entry.http
|
|
86
98
|
break
|
|
87
99
|
else
|
|
88
|
-
|
|
100
|
+
begin
|
|
101
|
+
entry.http.finish
|
|
102
|
+
rescue StandardError
|
|
103
|
+
nil
|
|
104
|
+
end
|
|
89
105
|
end
|
|
90
106
|
end
|
|
91
107
|
fresh
|
|
@@ -99,7 +115,11 @@ module Pinot
|
|
|
99
115
|
if pool_for_key.size < @max_pool_size
|
|
100
116
|
pool_for_key.push(PoolEntry.new(http, Process.clock_gettime(Process::CLOCK_MONOTONIC)))
|
|
101
117
|
else
|
|
102
|
-
|
|
118
|
+
begin
|
|
119
|
+
http.finish
|
|
120
|
+
rescue StandardError
|
|
121
|
+
nil
|
|
122
|
+
end
|
|
103
123
|
end
|
|
104
124
|
end
|
|
105
125
|
end
|
|
@@ -121,7 +141,11 @@ module Pinot
|
|
|
121
141
|
@pool.each_value do |entries|
|
|
122
142
|
entries.reject! do |entry|
|
|
123
143
|
if now - entry.checked_in_at >= @keep_alive_timeout
|
|
124
|
-
|
|
144
|
+
begin
|
|
145
|
+
entry.http.finish
|
|
146
|
+
rescue StandardError
|
|
147
|
+
nil
|
|
148
|
+
end
|
|
125
149
|
true
|
|
126
150
|
else
|
|
127
151
|
false
|
|
@@ -156,11 +180,11 @@ module Pinot
|
|
|
156
180
|
http.cert = OpenSSL::X509::Certificate.new(File.read(@tls_config.client_cert_file))
|
|
157
181
|
http.key = OpenSSL::PKey.read(File.read(@tls_config.client_key_file))
|
|
158
182
|
end
|
|
159
|
-
if @tls_config.insecure_skip_verify
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
183
|
+
http.verify_mode = if @tls_config.insecure_skip_verify
|
|
184
|
+
OpenSSL::SSL::VERIFY_NONE
|
|
185
|
+
else
|
|
186
|
+
OpenSSL::SSL::VERIFY_PEER
|
|
187
|
+
end
|
|
164
188
|
end
|
|
165
189
|
else
|
|
166
190
|
http.use_ssl = false
|
|
@@ -216,9 +240,9 @@ module Pinot
|
|
|
216
240
|
url = build_url(broker_address, request.query_format)
|
|
217
241
|
body = build_body(request)
|
|
218
242
|
headers = DEFAULT_HEADERS
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
243
|
+
.merge(@extra_headers)
|
|
244
|
+
.merge("X-Correlation-Id" => SecureRandom.uuid)
|
|
245
|
+
.merge(extra_request_headers)
|
|
222
246
|
|
|
223
247
|
resp = @http_client.post(url, body: body, headers: headers)
|
|
224
248
|
|
|
@@ -245,11 +269,11 @@ module Pinot
|
|
|
245
269
|
broker_response
|
|
246
270
|
rescue *RETRYABLE_HTTP_ERRORS, *RETRYABLE_ERRORS => e
|
|
247
271
|
if attempts < max_attempts
|
|
248
|
-
sleep_ms = (@retry_interval_ms || 200) * (2
|
|
272
|
+
sleep_ms = (@retry_interval_ms || 200) * (2**(attempts - 1))
|
|
249
273
|
sleep(sleep_ms / 1000.0)
|
|
250
274
|
retry
|
|
251
275
|
end
|
|
252
|
-
raise
|
|
276
|
+
raise(Net::ReadTimeout === e || Net::WriteTimeout === e ? QueryTimeoutError.new(e.message) : e)
|
|
253
277
|
end
|
|
254
278
|
end
|
|
255
279
|
|
|
@@ -282,9 +306,7 @@ module Pinot
|
|
|
282
306
|
if request.query_format == "sql"
|
|
283
307
|
parts << "groupByMode=sql;responseFormat=sql"
|
|
284
308
|
parts << "useMultistageEngine=true" if request.use_multistage_engine
|
|
285
|
-
if @timeout_ms && @timeout_ms > 0
|
|
286
|
-
parts << "timeoutMs=#{@timeout_ms}"
|
|
287
|
-
end
|
|
309
|
+
parts << "timeoutMs=#{@timeout_ms}" if @timeout_ms && @timeout_ms > 0
|
|
288
310
|
parts << "timeoutMs=#{request.query_timeout_ms}" if request.query_timeout_ms
|
|
289
311
|
end
|
|
290
312
|
parts.join(";")
|
data/lib/pinot/version.rb
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
require "json"
|
|
2
|
-
require "set"
|
|
3
2
|
require_relative "table_aware_broker_selector"
|
|
4
3
|
require_relative "errors"
|
|
5
4
|
|
|
6
5
|
module Pinot
|
|
7
6
|
class ZookeeperBrokerSelector < TableAwareBrokerSelector
|
|
8
7
|
# ZK path where Pinot stores broker external view
|
|
9
|
-
BROKER_EXTERNAL_VIEW_PATH = "/EXTERNALVIEW/brokerResource"
|
|
8
|
+
BROKER_EXTERNAL_VIEW_PATH = "/EXTERNALVIEW/brokerResource".freeze
|
|
10
9
|
|
|
11
10
|
def initialize(zk_path:, zk_client: nil)
|
|
12
11
|
super()
|
|
@@ -39,9 +38,13 @@ module Pinot
|
|
|
39
38
|
end
|
|
40
39
|
|
|
41
40
|
def setup_watcher
|
|
42
|
-
@zk.register(BROKER_EXTERNAL_VIEW_PATH) do |
|
|
43
|
-
|
|
44
|
-
|
|
41
|
+
@zk.register(BROKER_EXTERNAL_VIEW_PATH) do |_event|
|
|
42
|
+
begin
|
|
43
|
+
fetch_and_update
|
|
44
|
+
rescue StandardError
|
|
45
|
+
nil
|
|
46
|
+
end
|
|
47
|
+
setup_watcher # re-register watch after each trigger
|
|
45
48
|
end
|
|
46
49
|
# Set initial watch
|
|
47
50
|
@zk.exists?(BROKER_EXTERNAL_VIEW_PATH, watch: true)
|
|
@@ -61,12 +64,15 @@ module Pinot
|
|
|
61
64
|
brokers = []
|
|
62
65
|
broker_map.each do |broker_key, state|
|
|
63
66
|
next unless state == "ONLINE"
|
|
67
|
+
|
|
64
68
|
# Broker key format: Broker_<hostname>_<port>
|
|
65
69
|
# Use the last segment as port and the second-to-last as host
|
|
66
70
|
parts = broker_key.split("_")
|
|
67
71
|
next if parts.length < 2
|
|
72
|
+
|
|
68
73
|
port = parts.last
|
|
69
74
|
next unless port =~ /\A\d+\z/
|
|
75
|
+
|
|
70
76
|
host = parts[-2]
|
|
71
77
|
brokers << "#{host}:#{port}"
|
|
72
78
|
end
|
data/lib/pinot.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pinot-client
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.29.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Xiang Fu
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-04-
|
|
11
|
+
date: 2026-04-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: logger
|
|
@@ -94,6 +94,34 @@ dependencies:
|
|
|
94
94
|
- - "~>"
|
|
95
95
|
- !ruby/object:Gem::Version
|
|
96
96
|
version: '0.8'
|
|
97
|
+
- !ruby/object:Gem::Dependency
|
|
98
|
+
name: rubocop
|
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
|
100
|
+
requirements:
|
|
101
|
+
- - "~>"
|
|
102
|
+
- !ruby/object:Gem::Version
|
|
103
|
+
version: '1.65'
|
|
104
|
+
type: :development
|
|
105
|
+
prerelease: false
|
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
107
|
+
requirements:
|
|
108
|
+
- - "~>"
|
|
109
|
+
- !ruby/object:Gem::Version
|
|
110
|
+
version: '1.65'
|
|
111
|
+
- !ruby/object:Gem::Dependency
|
|
112
|
+
name: rubocop-rspec
|
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
|
114
|
+
requirements:
|
|
115
|
+
- - "~>"
|
|
116
|
+
- !ruby/object:Gem::Version
|
|
117
|
+
version: '3.0'
|
|
118
|
+
type: :development
|
|
119
|
+
prerelease: false
|
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
121
|
+
requirements:
|
|
122
|
+
- - "~>"
|
|
123
|
+
- !ruby/object:Gem::Version
|
|
124
|
+
version: '3.0'
|
|
97
125
|
description: A Ruby client for Apache Pinot, mirroring the Go client API
|
|
98
126
|
email:
|
|
99
127
|
executables: []
|
|
@@ -116,6 +144,7 @@ files:
|
|
|
116
144
|
- lib/pinot/grpc_transport.rb
|
|
117
145
|
- lib/pinot/instrumentation.rb
|
|
118
146
|
- lib/pinot/logger.rb
|
|
147
|
+
- lib/pinot/open_telemetry.rb
|
|
119
148
|
- lib/pinot/paginator.rb
|
|
120
149
|
- lib/pinot/prepared_statement.rb
|
|
121
150
|
- lib/pinot/proto/broker_service.proto
|