pinot-client 1.28.0 → 1.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/pinot/active_support_notifications.rb +4 -4
- data/lib/pinot/circuit_breaker.rb +5 -4
- data/lib/pinot/config.rb +1 -3
- data/lib/pinot/connection.rb +45 -18
- data/lib/pinot/connection_factory.rb +1 -1
- data/lib/pinot/controller_based_broker_selector.rb +6 -9
- data/lib/pinot/grpc_transport.rb +1 -1
- data/lib/pinot/instrumentation.rb +1 -1
- data/lib/pinot/open_telemetry.rb +8 -7
- data/lib/pinot/paginator.rb +3 -1
- data/lib/pinot/prepared_statement.rb +11 -12
- data/lib/pinot/proto/broker_service_pb.rb +3 -3
- data/lib/pinot/proto/broker_service_services_pb.rb +3 -4
- data/lib/pinot/response.rb +22 -8
- data/lib/pinot/simple_broker_selector.rb +1 -0
- data/lib/pinot/table_aware_broker_selector.rb +4 -2
- data/lib/pinot/transport.rb +42 -20
- data/lib/pinot/version.rb +1 -1
- data/lib/pinot/zookeeper_broker_selector.rb +11 -5
- data/lib/pinot.rb +1 -1
- metadata +29 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f728a6fe054a729893524d1da24e6ab2cddabda2d17c3bfae183e77271a70b1c
|
|
4
|
+
data.tar.gz: 6f10e6f81f725d9e34ed0d68a3ed04070244308f949a7a01533f90940fff18a9
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 26d0a240f8d335f1acf02363ab974bf8e0323c3ddc045b2c9539340468231d68c6fe6c1c8517e0fb8a79ad4b154bd8178847e890d8475280200b33f327fcb467
|
|
7
|
+
data.tar.gz: 987c72674f5902f79365ff493ff541299e0efcc13374016bee1c52b98018734ff50498d7991c08807f50c5e0b70aff2be734eaa564ba7aa640da6530e66b4665
|
|
@@ -43,7 +43,7 @@ module Pinot
|
|
|
43
43
|
# ActiveSupport::Notifications to already be defined at install! time (which
|
|
44
44
|
# is always the case in a Rails process).
|
|
45
45
|
module ActiveSupportNotifications
|
|
46
|
-
EVENT_NAME = "sql.pinot"
|
|
46
|
+
EVENT_NAME = "sql.pinot".freeze
|
|
47
47
|
|
|
48
48
|
def self.install!
|
|
49
49
|
return if installed?
|
|
@@ -64,10 +64,10 @@ module Pinot
|
|
|
64
64
|
|
|
65
65
|
def self.notify(event)
|
|
66
66
|
payload = {
|
|
67
|
-
sql:
|
|
68
|
-
name:
|
|
67
|
+
sql: event[:query],
|
|
68
|
+
name: event[:table],
|
|
69
69
|
duration: event[:duration_ms],
|
|
70
|
-
success:
|
|
70
|
+
success: event[:success]
|
|
71
71
|
}
|
|
72
72
|
|
|
73
73
|
if (err = event[:error])
|
|
@@ -34,7 +34,8 @@ module Pinot
|
|
|
34
34
|
OPEN = :open
|
|
35
35
|
HALF_OPEN = :half_open
|
|
36
36
|
|
|
37
|
-
BrokerCircuitOpenError
|
|
37
|
+
class BrokerCircuitOpenError < BrokerNotFoundError
|
|
38
|
+
end
|
|
38
39
|
|
|
39
40
|
attr_reader :state, :failure_count
|
|
40
41
|
|
|
@@ -48,14 +49,14 @@ module Pinot
|
|
|
48
49
|
end
|
|
49
50
|
|
|
50
51
|
# Call the block; record success/failure and enforce open-circuit rejection.
|
|
51
|
-
def call(
|
|
52
|
+
def call(_broker_address)
|
|
52
53
|
@mutex.synchronize { check_state! }
|
|
53
54
|
begin
|
|
54
55
|
result = yield
|
|
55
56
|
@mutex.synchronize { on_success }
|
|
56
57
|
result
|
|
57
58
|
rescue BrokerUnavailableError, Errno::ECONNRESET, Errno::ECONNREFUSED,
|
|
58
|
-
Errno::ETIMEDOUT, Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout
|
|
59
|
+
Errno::ETIMEDOUT, Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout
|
|
59
60
|
@mutex.synchronize { on_failure }
|
|
60
61
|
raise
|
|
61
62
|
end
|
|
@@ -126,7 +127,7 @@ module Pinot
|
|
|
126
127
|
@mutex.synchronize do
|
|
127
128
|
@breakers[broker_address] ||= CircuitBreaker.new(
|
|
128
129
|
failure_threshold: @failure_threshold,
|
|
129
|
-
open_timeout:
|
|
130
|
+
open_timeout: @open_timeout
|
|
130
131
|
)
|
|
131
132
|
end
|
|
132
133
|
end
|
data/lib/pinot/config.rb
CHANGED
|
@@ -89,9 +89,7 @@ module Pinot
|
|
|
89
89
|
raise ConfigurationError, "query_timeout_ms must be positive, got: #{query_timeout_ms}"
|
|
90
90
|
end
|
|
91
91
|
|
|
92
|
-
if !pool_size.nil? && pool_size < 1
|
|
93
|
-
raise ConfigurationError, "pool_size must be at least 1, got: #{pool_size}"
|
|
94
|
-
end
|
|
92
|
+
raise ConfigurationError, "pool_size must be at least 1, got: #{pool_size}" if !pool_size.nil? && pool_size < 1
|
|
95
93
|
|
|
96
94
|
if !keep_alive_timeout.nil? && keep_alive_timeout <= 0
|
|
97
95
|
raise ConfigurationError, "keep_alive_timeout must be positive, got: #{keep_alive_timeout}"
|
data/lib/pinot/connection.rb
CHANGED
|
@@ -34,9 +34,7 @@ module Pinot
|
|
|
34
34
|
@circuit_breaker_registry = circuit_breaker_registry
|
|
35
35
|
end
|
|
36
36
|
|
|
37
|
-
|
|
38
|
-
@use_multistage_engine = val
|
|
39
|
-
end
|
|
37
|
+
attr_writer :use_multistage_engine
|
|
40
38
|
|
|
41
39
|
def open_trace
|
|
42
40
|
@trace = true
|
|
@@ -91,8 +89,6 @@ module Pinot
|
|
|
91
89
|
def execute_sql_with_params(table, query_pattern, params, query_timeout_ms: nil, headers: {})
|
|
92
90
|
query = format_query(query_pattern, params)
|
|
93
91
|
execute_sql(table, query, query_timeout_ms: query_timeout_ms, headers: headers)
|
|
94
|
-
rescue => e
|
|
95
|
-
raise e
|
|
96
92
|
end
|
|
97
93
|
|
|
98
94
|
# Execute multiple queries in parallel and return results in the same order.
|
|
@@ -121,7 +117,7 @@ module Pinot
|
|
|
121
117
|
def execute_many(queries, max_concurrency: nil)
|
|
122
118
|
return [] if queries.empty?
|
|
123
119
|
|
|
124
|
-
results
|
|
120
|
+
results = Array.new(queries.size)
|
|
125
121
|
# Queue acts as a counting semaphore: pre-filled with N tokens.
|
|
126
122
|
sem = max_concurrency ? build_semaphore(max_concurrency) : nil
|
|
127
123
|
|
|
@@ -131,14 +127,14 @@ module Pinot
|
|
|
131
127
|
timeout_ms = item[:query_timeout_ms] || item["query_timeout_ms"]
|
|
132
128
|
|
|
133
129
|
Thread.new do
|
|
134
|
-
sem&.pop
|
|
130
|
+
sem&.pop # acquire
|
|
135
131
|
begin
|
|
136
132
|
resp = execute_sql(table, query, query_timeout_ms: timeout_ms)
|
|
137
133
|
results[idx] = QueryResult.new(table: table, query: query, response: resp, error: nil)
|
|
138
|
-
rescue => e
|
|
134
|
+
rescue StandardError => e
|
|
139
135
|
results[idx] = QueryResult.new(table: table, query: query, response: nil, error: e)
|
|
140
136
|
ensure
|
|
141
|
-
sem&.push(:token)
|
|
137
|
+
sem&.push(:token) # release
|
|
142
138
|
end
|
|
143
139
|
end
|
|
144
140
|
end
|
|
@@ -166,11 +162,42 @@ module Pinot
|
|
|
166
162
|
@transport.http_client,
|
|
167
163
|
broker,
|
|
168
164
|
query,
|
|
169
|
-
page_size:
|
|
165
|
+
page_size: page_size,
|
|
170
166
|
extra_headers: extra_headers
|
|
171
167
|
)
|
|
172
168
|
end
|
|
173
169
|
|
|
170
|
+
# Check whether a broker is reachable and responding to queries.
|
|
171
|
+
#
|
|
172
|
+
# Runs a lightweight broker-side liveness check: first tries the dedicated
|
|
173
|
+
# +/health+ HTTP endpoint (returns 200 when the broker is healthy), and falls
|
|
174
|
+
# back to executing "SELECT 1 FROM DUAL" if the endpoint is unavailable.
|
|
175
|
+
#
|
|
176
|
+
# Returns +true+ when the broker responds successfully, +false+ on any
|
|
177
|
+
# error (connection refused, timeout, non-200 response, etc.). Never raises.
|
|
178
|
+
#
|
|
179
|
+
# Intended for Kubernetes readiness / liveness probes and health-check
|
|
180
|
+
# endpoints in Rails / Rack applications:
|
|
181
|
+
#
|
|
182
|
+
# get "/healthz" do
|
|
183
|
+
# conn.healthy? ? [200, "OK"] : [503, "Pinot unavailable"]
|
|
184
|
+
# end
|
|
185
|
+
#
|
|
186
|
+
# @param table [String, nil] table used for broker selection (nil = any broker)
|
|
187
|
+
# @param timeout_ms [Integer] per-check timeout in ms (default 2000)
|
|
188
|
+
# @return [Boolean]
|
|
189
|
+
def healthy?(table: nil, timeout_ms: 2_000)
|
|
190
|
+
broker = @broker_selector.select_broker(table || "")
|
|
191
|
+
base = broker.start_with?("http://", "https://") ? broker : "http://#{broker}"
|
|
192
|
+
client = HttpClient.new(timeout: timeout_ms / 1000.0)
|
|
193
|
+
resp = client.get("#{base}/health", headers: {})
|
|
194
|
+
resp.code.to_i == 200
|
|
195
|
+
rescue StandardError
|
|
196
|
+
false
|
|
197
|
+
ensure
|
|
198
|
+
client&.close
|
|
199
|
+
end
|
|
200
|
+
|
|
174
201
|
# Create a PreparedStatement from a query template with +?+ placeholders.
|
|
175
202
|
#
|
|
176
203
|
# stmt = conn.prepare("myTable", "SELECT * FROM myTable WHERE id = ? AND name = ?")
|
|
@@ -185,8 +212,10 @@ module Pinot
|
|
|
185
212
|
def prepare(table, query_template)
|
|
186
213
|
raise ArgumentError, "table name cannot be empty" if table.nil? || table.strip.empty?
|
|
187
214
|
raise ArgumentError, "query template cannot be empty" if query_template.nil? || query_template.strip.empty?
|
|
215
|
+
|
|
188
216
|
count = query_template.count("?")
|
|
189
217
|
raise ArgumentError, "query template must contain at least one parameter placeholder (?)" if count == 0
|
|
218
|
+
|
|
190
219
|
PreparedStatementImpl.new(connection: self, table: table, query_template: query_template)
|
|
191
220
|
end
|
|
192
221
|
|
|
@@ -196,12 +225,13 @@ module Pinot
|
|
|
196
225
|
if placeholders != params.length
|
|
197
226
|
raise "failed to format query: number of placeholders in queryPattern (#{placeholders}) does not match number of params (#{params.length})"
|
|
198
227
|
end
|
|
228
|
+
|
|
199
229
|
parts = pattern.split("?", -1)
|
|
200
230
|
result = ""
|
|
201
231
|
params.each_with_index do |param, i|
|
|
202
232
|
formatted = begin
|
|
203
233
|
format_arg(param)
|
|
204
|
-
rescue => e
|
|
234
|
+
rescue StandardError => e
|
|
205
235
|
raise "failed to format query: failed to format parameter: #{e.message}"
|
|
206
236
|
end
|
|
207
237
|
result += parts[i] + formatted
|
|
@@ -213,11 +243,7 @@ module Pinot
|
|
|
213
243
|
case value
|
|
214
244
|
when String
|
|
215
245
|
"'#{value.gsub("'", "''")}'"
|
|
216
|
-
when Integer
|
|
217
|
-
value.to_s
|
|
218
|
-
when Float
|
|
219
|
-
value.to_s
|
|
220
|
-
when TrueClass, FalseClass
|
|
246
|
+
when Integer, Float, TrueClass, FalseClass
|
|
221
247
|
value.to_s
|
|
222
248
|
when BigDecimal
|
|
223
249
|
s = value.to_s("F")
|
|
@@ -239,9 +265,10 @@ module Pinot
|
|
|
239
265
|
q
|
|
240
266
|
end
|
|
241
267
|
|
|
242
|
-
def run_with_circuit_breaker(broker, &
|
|
268
|
+
def run_with_circuit_breaker(broker, &)
|
|
243
269
|
return yield unless @circuit_breaker_registry
|
|
244
|
-
|
|
270
|
+
|
|
271
|
+
@circuit_breaker_registry.for(broker).call(broker, &)
|
|
245
272
|
end
|
|
246
273
|
|
|
247
274
|
def logger
|
|
@@ -138,7 +138,7 @@ module Pinot
|
|
|
138
138
|
|
|
139
139
|
CircuitBreakerRegistry.new(
|
|
140
140
|
failure_threshold: config.circuit_breaker_threshold || 5,
|
|
141
|
-
open_timeout:
|
|
141
|
+
open_timeout: config.circuit_breaker_timeout || 30
|
|
142
142
|
)
|
|
143
143
|
end
|
|
144
144
|
private_class_method :build_circuit_breaker_registry
|
|
@@ -4,7 +4,7 @@ require "json"
|
|
|
4
4
|
|
|
5
5
|
module Pinot
|
|
6
6
|
class ControllerBasedBrokerSelector < TableAwareBrokerSelector
|
|
7
|
-
CONTROLLER_API_PATH = "/v2/brokers/tables?state=ONLINE"
|
|
7
|
+
CONTROLLER_API_PATH = "/v2/brokers/tables?state=ONLINE".freeze
|
|
8
8
|
DEFAULT_UPDATE_FREQ_MS = 1000
|
|
9
9
|
|
|
10
10
|
def initialize(config, http_client = nil, logger: nil)
|
|
@@ -27,9 +27,8 @@ module Pinot
|
|
|
27
27
|
addr = address.to_s
|
|
28
28
|
if addr.include?("://")
|
|
29
29
|
scheme = addr.split("://").first
|
|
30
|
-
unless %w[http https].include?(scheme)
|
|
31
|
-
|
|
32
|
-
end
|
|
30
|
+
raise ConfigurationError, "unsupported controller URL scheme: #{scheme}" unless %w[http https].include?(scheme)
|
|
31
|
+
|
|
33
32
|
addr.chomp("/") + CONTROLLER_API_PATH
|
|
34
33
|
else
|
|
35
34
|
"http://#{addr.chomp("/")}#{CONTROLLER_API_PATH}"
|
|
@@ -40,13 +39,11 @@ module Pinot
|
|
|
40
39
|
|
|
41
40
|
def fetch_and_update
|
|
42
41
|
headers = { "Accept" => "application/json" }
|
|
43
|
-
|
|
42
|
+
.merge(@config.extra_controller_api_headers || {})
|
|
44
43
|
|
|
45
44
|
resp = @internal_http.get(@controller_url, headers: headers)
|
|
46
45
|
|
|
47
|
-
unless resp.code.to_i == 200
|
|
48
|
-
raise TransportError, "controller API returned HTTP status code #{resp.code}"
|
|
49
|
-
end
|
|
46
|
+
raise TransportError, "controller API returned HTTP status code #{resp.code}" unless resp.code.to_i == 200
|
|
50
47
|
|
|
51
48
|
body = resp.body
|
|
52
49
|
begin
|
|
@@ -70,7 +67,7 @@ module Pinot
|
|
|
70
67
|
sleep interval
|
|
71
68
|
begin
|
|
72
69
|
fetch_and_update
|
|
73
|
-
rescue => e
|
|
70
|
+
rescue StandardError => e
|
|
74
71
|
logger.warn "Pinot controller refresh failed: #{e.message}"
|
|
75
72
|
end
|
|
76
73
|
end
|
data/lib/pinot/grpc_transport.rb
CHANGED
|
@@ -92,7 +92,7 @@ module Pinot
|
|
|
92
92
|
duration_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000
|
|
93
93
|
notify(table: table, query: query, duration_ms: duration_ms, success: true, error: nil)
|
|
94
94
|
result
|
|
95
|
-
rescue => e
|
|
95
|
+
rescue StandardError => e
|
|
96
96
|
duration_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000
|
|
97
97
|
notify(table: table, query: query, duration_ms: duration_ms, success: false, error: e)
|
|
98
98
|
raise
|
data/lib/pinot/open_telemetry.rb
CHANGED
|
@@ -49,9 +49,9 @@ module Pinot
|
|
|
49
49
|
# Note: this gem does NOT depend on opentelemetry-api or opentelemetry-sdk.
|
|
50
50
|
# Both must be present and initialized before install! is called.
|
|
51
51
|
module OpenTelemetry
|
|
52
|
-
SPAN_NAME = "pinot.query"
|
|
53
|
-
DB_SYSTEM = "pinot"
|
|
54
|
-
TRACER_NAME = "pinot-client"
|
|
52
|
+
SPAN_NAME = "pinot.query".freeze
|
|
53
|
+
DB_SYSTEM = "pinot".freeze
|
|
54
|
+
TRACER_NAME = "pinot-client".freeze
|
|
55
55
|
|
|
56
56
|
@installed = false
|
|
57
57
|
@enabled = true
|
|
@@ -80,7 +80,7 @@ module Pinot
|
|
|
80
80
|
def self.uninstall!
|
|
81
81
|
::Pinot::Instrumentation.around = nil
|
|
82
82
|
@installed = false
|
|
83
|
-
#
|
|
83
|
+
# NOTE: JsonHttpTransport prepend is permanent once applied (Ruby limitation).
|
|
84
84
|
# Disable the propagator by unsetting the flag — it no-ops when disabled.
|
|
85
85
|
end
|
|
86
86
|
|
|
@@ -111,7 +111,7 @@ module Pinot
|
|
|
111
111
|
table: table, query: query, duration_ms: duration_ms, success: true, error: nil
|
|
112
112
|
)
|
|
113
113
|
result
|
|
114
|
-
rescue => e
|
|
114
|
+
rescue StandardError => e
|
|
115
115
|
duration_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000
|
|
116
116
|
span.record_exception(e)
|
|
117
117
|
span.status = ::OpenTelemetry::Trace::Status.error(e.message)
|
|
@@ -126,9 +126,9 @@ module Pinot
|
|
|
126
126
|
|
|
127
127
|
def self._span_attributes(table, query)
|
|
128
128
|
attrs = {
|
|
129
|
-
"db.system"
|
|
129
|
+
"db.system" => DB_SYSTEM,
|
|
130
130
|
"db.statement" => query,
|
|
131
|
-
"db.name"
|
|
131
|
+
"db.name" => table.to_s
|
|
132
132
|
}
|
|
133
133
|
op = query.to_s.lstrip.split(/\s+/, 2).first&.upcase
|
|
134
134
|
attrs["db.operation"] = op if op && !op.empty?
|
|
@@ -140,6 +140,7 @@ module Pinot
|
|
|
140
140
|
# outbound HTTP request. Applied once via Module#prepend.
|
|
141
141
|
def self._patch_transport
|
|
142
142
|
return if ::Pinot::JsonHttpTransport.ancestors.include?(TraceContextInjector)
|
|
143
|
+
|
|
143
144
|
::Pinot::JsonHttpTransport.prepend(TraceContextInjector)
|
|
144
145
|
end
|
|
145
146
|
private_class_method :_patch_transport
|
data/lib/pinot/paginator.rb
CHANGED
|
@@ -53,7 +53,7 @@ module Pinot
|
|
|
53
53
|
@extra_headers = extra_headers
|
|
54
54
|
|
|
55
55
|
@request_id = nil
|
|
56
|
-
@cursor_base = nil
|
|
56
|
+
@cursor_base = nil # "http://host:port" — set after first response
|
|
57
57
|
@exhausted = false
|
|
58
58
|
end
|
|
59
59
|
|
|
@@ -136,11 +136,13 @@ module Pinot
|
|
|
136
136
|
|
|
137
137
|
def broker_base(address)
|
|
138
138
|
return address if address.start_with?("http://", "https://")
|
|
139
|
+
|
|
139
140
|
"http://#{address}"
|
|
140
141
|
end
|
|
141
142
|
|
|
142
143
|
def broker_base_from_response(resp)
|
|
143
144
|
return nil unless resp.broker_host && resp.broker_port
|
|
145
|
+
|
|
144
146
|
"http://#{resp.broker_host}:#{resp.broker_port}"
|
|
145
147
|
end
|
|
146
148
|
|
|
@@ -52,9 +52,8 @@ module Pinot
|
|
|
52
52
|
def set(index, value)
|
|
53
53
|
@mutex.synchronize do
|
|
54
54
|
raise PreparedStatementClosedError, "prepared statement is closed" if @closed
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
end
|
|
55
|
+
raise "parameter index #{index} is out of range [1, #{@param_count}]" unless index.between?(1, @param_count)
|
|
56
|
+
|
|
58
57
|
@parameters[index - 1] = value
|
|
59
58
|
end
|
|
60
59
|
nil
|
|
@@ -63,13 +62,14 @@ module Pinot
|
|
|
63
62
|
def execute(headers: {})
|
|
64
63
|
@mutex.synchronize do
|
|
65
64
|
raise PreparedStatementClosedError, "prepared statement is closed" if @closed
|
|
65
|
+
|
|
66
66
|
@parameters.each_with_index do |p, i|
|
|
67
67
|
raise "parameter at index #{i + 1} is not set" if p.nil?
|
|
68
68
|
end
|
|
69
69
|
end
|
|
70
70
|
query = begin
|
|
71
71
|
build_query(@parameters)
|
|
72
|
-
rescue => e
|
|
72
|
+
rescue StandardError => e
|
|
73
73
|
raise "failed to build query: #{e.message}"
|
|
74
74
|
end
|
|
75
75
|
@connection.execute_sql(@table, query, headers: headers)
|
|
@@ -77,12 +77,11 @@ module Pinot
|
|
|
77
77
|
|
|
78
78
|
def execute_with_params(*params, headers: {})
|
|
79
79
|
@mutex.synchronize { raise PreparedStatementClosedError, "prepared statement is closed" if @closed }
|
|
80
|
-
if params.length != @param_count
|
|
81
|
-
|
|
82
|
-
end
|
|
80
|
+
raise "expected #{@param_count} parameters, got #{params.length}" if params.length != @param_count
|
|
81
|
+
|
|
83
82
|
query = begin
|
|
84
83
|
build_query(params)
|
|
85
|
-
rescue => e
|
|
84
|
+
rescue StandardError => e
|
|
86
85
|
raise "failed to build query: #{e.message}"
|
|
87
86
|
end
|
|
88
87
|
@connection.execute_sql(@table, query, headers: headers)
|
|
@@ -91,6 +90,7 @@ module Pinot
|
|
|
91
90
|
def clear_parameters
|
|
92
91
|
@mutex.synchronize do
|
|
93
92
|
raise PreparedStatementClosedError, "prepared statement is closed" if @closed
|
|
93
|
+
|
|
94
94
|
@parameters.fill(nil)
|
|
95
95
|
end
|
|
96
96
|
nil
|
|
@@ -105,14 +105,13 @@ module Pinot
|
|
|
105
105
|
end
|
|
106
106
|
|
|
107
107
|
def build_query(params)
|
|
108
|
-
if params.length != @param_count
|
|
109
|
-
|
|
110
|
-
end
|
|
108
|
+
raise "expected #{@param_count} parameters, got #{params.length}" if params.length != @param_count
|
|
109
|
+
|
|
111
110
|
result = ""
|
|
112
111
|
params.each_with_index do |param, i|
|
|
113
112
|
formatted = begin
|
|
114
113
|
@connection.format_arg(param)
|
|
115
|
-
rescue => e
|
|
114
|
+
rescue StandardError => e
|
|
116
115
|
raise "failed to format parameter: #{e.message}"
|
|
117
116
|
end
|
|
118
117
|
result += @parts[i] + formatted
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
+
|
|
2
3
|
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
3
4
|
# source: broker_service.proto
|
|
4
5
|
|
|
5
|
-
require
|
|
6
|
-
|
|
6
|
+
require "google/protobuf"
|
|
7
7
|
|
|
8
8
|
descriptor_data = "\n\x14\x62roker_service.proto\x12\x11pinot.broker.grpc\"\x8f\x01\n\rBrokerRequest\x12\x0b\n\x03sql\x18\x01 \x01(\t\x12@\n\x08metadata\x18\x02 \x03(\x0b\x32..pinot.broker.grpc.BrokerRequest.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\":\n\x0e\x42rokerResponse\x12\x17\n\x0fresult_row_size\x18\x01 \x01(\x05\x12\x0f\n\x07payload\x18\x02 \x01(\x0c\x32m\n\x1cPinotClientGrpcBrokerService\x12M\n\x06Submit\x12 .pinot.broker.grpc.BrokerRequest\x1a!.pinot.broker.grpc.BrokerResponseb\x06proto3"
|
|
9
9
|
|
|
10
|
-
pool =
|
|
10
|
+
pool = Google::Protobuf::DescriptorPool.generated_pool
|
|
11
11
|
pool.add_serialized_file(descriptor_data)
|
|
12
12
|
|
|
13
13
|
module Pinot
|
|
@@ -1,20 +1,19 @@
|
|
|
1
1
|
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
2
2
|
# Source: broker_service.proto for package 'pinot.broker.grpc'
|
|
3
3
|
|
|
4
|
-
require
|
|
5
|
-
require_relative
|
|
4
|
+
require "grpc"
|
|
5
|
+
require_relative "broker_service_pb"
|
|
6
6
|
|
|
7
7
|
module Pinot
|
|
8
8
|
module Broker
|
|
9
9
|
module Grpc
|
|
10
10
|
module PinotClientGrpcBrokerService
|
|
11
11
|
class Service
|
|
12
|
-
|
|
13
12
|
include ::GRPC::GenericService
|
|
14
13
|
|
|
15
14
|
self.marshal_class_method = :encode
|
|
16
15
|
self.unmarshal_class_method = :decode
|
|
17
|
-
self.service_name =
|
|
16
|
+
self.service_name = "pinot.broker.grpc.PinotClientGrpcBrokerService"
|
|
18
17
|
|
|
19
18
|
rpc :Submit, ::Pinot::Broker::Grpc::BrokerRequest, ::Pinot::Broker::Grpc::BrokerResponse
|
|
20
19
|
end
|
data/lib/pinot/response.rb
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
require "bigdecimal"
|
|
2
2
|
|
|
3
3
|
module Pinot
|
|
4
|
-
INT32_MAX =
|
|
4
|
+
INT32_MAX = 2_147_483_647
|
|
5
5
|
INT32_MIN = -2_147_483_648
|
|
6
|
-
INT64_MAX =
|
|
6
|
+
INT64_MAX = 9_223_372_036_854_775_807
|
|
7
7
|
INT64_MIN = -9_223_372_036_854_775_808
|
|
8
8
|
FLOAT32_MAX = 3.4028235e+38
|
|
9
9
|
|
|
@@ -110,14 +110,20 @@ module Pinot
|
|
|
110
110
|
if raw.include?(".") || raw.include?("e") || raw.include?("E")
|
|
111
111
|
# Floating point string — check if it's a whole number
|
|
112
112
|
bd = BigDecimal(raw)
|
|
113
|
-
|
|
113
|
+
begin
|
|
114
|
+
return 0 if bd.infinite? || bd.nan?
|
|
115
|
+
rescue StandardError
|
|
116
|
+
return 0
|
|
117
|
+
end
|
|
114
118
|
int_val = bd.to_i
|
|
115
119
|
return 0 unless bd == BigDecimal(int_val.to_s)
|
|
116
120
|
return 0 if int_val > INT32_MAX || int_val < INT32_MIN
|
|
121
|
+
|
|
117
122
|
int_val.to_i
|
|
118
123
|
else
|
|
119
124
|
int_val = Integer(raw)
|
|
120
125
|
return 0 if int_val > INT32_MAX || int_val < INT32_MIN
|
|
126
|
+
|
|
121
127
|
int_val
|
|
122
128
|
end
|
|
123
129
|
rescue ArgumentError, TypeError
|
|
@@ -133,16 +139,21 @@ module Pinot
|
|
|
133
139
|
begin
|
|
134
140
|
if raw.include?(".") || raw.include?("e") || raw.include?("E")
|
|
135
141
|
bd = BigDecimal(raw)
|
|
136
|
-
|
|
142
|
+
begin
|
|
143
|
+
return 0 if bd.infinite? || bd.nan?
|
|
144
|
+
rescue StandardError
|
|
145
|
+
return 0
|
|
146
|
+
end
|
|
137
147
|
int_val = bd.to_i
|
|
138
148
|
return 0 unless bd == BigDecimal(int_val.to_s)
|
|
139
|
-
|
|
140
|
-
int_val
|
|
149
|
+
|
|
141
150
|
else
|
|
142
151
|
int_val = Integer(raw)
|
|
143
|
-
|
|
144
|
-
int_val
|
|
152
|
+
|
|
145
153
|
end
|
|
154
|
+
return 0 if int_val > INT64_MAX || int_val < INT64_MIN
|
|
155
|
+
|
|
156
|
+
int_val
|
|
146
157
|
rescue ArgumentError, TypeError
|
|
147
158
|
0
|
|
148
159
|
end
|
|
@@ -156,8 +167,10 @@ module Pinot
|
|
|
156
167
|
begin
|
|
157
168
|
f = Float(raw)
|
|
158
169
|
return 0.0 if f.infinite? || f.nan?
|
|
170
|
+
|
|
159
171
|
f32 = f.to_f
|
|
160
172
|
return 0.0 if f32.abs > FLOAT32_MAX
|
|
173
|
+
|
|
161
174
|
f32
|
|
162
175
|
rescue ArgumentError, TypeError
|
|
163
176
|
0.0
|
|
@@ -172,6 +185,7 @@ module Pinot
|
|
|
172
185
|
begin
|
|
173
186
|
f = Float(raw)
|
|
174
187
|
return 0.0 if f.infinite? || f.nan?
|
|
188
|
+
|
|
175
189
|
f
|
|
176
190
|
rescue ArgumentError, TypeError
|
|
177
191
|
0.0
|
|
@@ -2,8 +2,8 @@ module Pinot
|
|
|
2
2
|
class TableAwareBrokerSelector
|
|
3
3
|
include BrokerSelector
|
|
4
4
|
|
|
5
|
-
OFFLINE_SUFFIX = "_OFFLINE"
|
|
6
|
-
REALTIME_SUFFIX = "_REALTIME"
|
|
5
|
+
OFFLINE_SUFFIX = "_OFFLINE".freeze
|
|
6
|
+
REALTIME_SUFFIX = "_REALTIME".freeze
|
|
7
7
|
|
|
8
8
|
def initialize
|
|
9
9
|
@mutex = Mutex.new
|
|
@@ -20,11 +20,13 @@ module Pinot
|
|
|
20
20
|
@mutex.synchronize do
|
|
21
21
|
if table_name.empty?
|
|
22
22
|
raise BrokerNotFoundError, "no available broker" if @all_broker_list.empty?
|
|
23
|
+
|
|
23
24
|
return @all_broker_list.sample
|
|
24
25
|
end
|
|
25
26
|
brokers = @table_broker_map[table_name]
|
|
26
27
|
raise TableNotFoundError, "unable to find table: #{table}" unless brokers
|
|
27
28
|
raise BrokerNotFoundError, "no available broker for table: #{table}" if brokers.empty?
|
|
29
|
+
|
|
28
30
|
brokers.sample
|
|
29
31
|
end
|
|
30
32
|
end
|
data/lib/pinot/transport.rb
CHANGED
|
@@ -50,10 +50,18 @@ module Pinot
|
|
|
50
50
|
end
|
|
51
51
|
|
|
52
52
|
def close
|
|
53
|
-
|
|
53
|
+
begin
|
|
54
|
+
@reaper.kill
|
|
55
|
+
rescue StandardError
|
|
56
|
+
nil
|
|
57
|
+
end
|
|
54
58
|
@pool_mutex.synchronize do
|
|
55
59
|
@pool.each_value do |entries|
|
|
56
|
-
entries.each
|
|
60
|
+
entries.each do |entry|
|
|
61
|
+
entry.http.finish
|
|
62
|
+
rescue StandardError
|
|
63
|
+
nil
|
|
64
|
+
end
|
|
57
65
|
end
|
|
58
66
|
@pool.clear
|
|
59
67
|
end
|
|
@@ -69,8 +77,12 @@ module Pinot
|
|
|
69
77
|
result = yield http
|
|
70
78
|
checkin(key, http)
|
|
71
79
|
result
|
|
72
|
-
rescue => e
|
|
73
|
-
|
|
80
|
+
rescue StandardError => e
|
|
81
|
+
begin
|
|
82
|
+
http.finish
|
|
83
|
+
rescue StandardError
|
|
84
|
+
nil
|
|
85
|
+
end
|
|
74
86
|
raise e
|
|
75
87
|
end
|
|
76
88
|
end
|
|
@@ -85,7 +97,11 @@ module Pinot
|
|
|
85
97
|
fresh = entry.http
|
|
86
98
|
break
|
|
87
99
|
else
|
|
88
|
-
|
|
100
|
+
begin
|
|
101
|
+
entry.http.finish
|
|
102
|
+
rescue StandardError
|
|
103
|
+
nil
|
|
104
|
+
end
|
|
89
105
|
end
|
|
90
106
|
end
|
|
91
107
|
fresh
|
|
@@ -99,7 +115,11 @@ module Pinot
|
|
|
99
115
|
if pool_for_key.size < @max_pool_size
|
|
100
116
|
pool_for_key.push(PoolEntry.new(http, Process.clock_gettime(Process::CLOCK_MONOTONIC)))
|
|
101
117
|
else
|
|
102
|
-
|
|
118
|
+
begin
|
|
119
|
+
http.finish
|
|
120
|
+
rescue StandardError
|
|
121
|
+
nil
|
|
122
|
+
end
|
|
103
123
|
end
|
|
104
124
|
end
|
|
105
125
|
end
|
|
@@ -121,7 +141,11 @@ module Pinot
|
|
|
121
141
|
@pool.each_value do |entries|
|
|
122
142
|
entries.reject! do |entry|
|
|
123
143
|
if now - entry.checked_in_at >= @keep_alive_timeout
|
|
124
|
-
|
|
144
|
+
begin
|
|
145
|
+
entry.http.finish
|
|
146
|
+
rescue StandardError
|
|
147
|
+
nil
|
|
148
|
+
end
|
|
125
149
|
true
|
|
126
150
|
else
|
|
127
151
|
false
|
|
@@ -156,11 +180,11 @@ module Pinot
|
|
|
156
180
|
http.cert = OpenSSL::X509::Certificate.new(File.read(@tls_config.client_cert_file))
|
|
157
181
|
http.key = OpenSSL::PKey.read(File.read(@tls_config.client_key_file))
|
|
158
182
|
end
|
|
159
|
-
if @tls_config.insecure_skip_verify
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
183
|
+
http.verify_mode = if @tls_config.insecure_skip_verify
|
|
184
|
+
OpenSSL::SSL::VERIFY_NONE
|
|
185
|
+
else
|
|
186
|
+
OpenSSL::SSL::VERIFY_PEER
|
|
187
|
+
end
|
|
164
188
|
end
|
|
165
189
|
else
|
|
166
190
|
http.use_ssl = false
|
|
@@ -216,9 +240,9 @@ module Pinot
|
|
|
216
240
|
url = build_url(broker_address, request.query_format)
|
|
217
241
|
body = build_body(request)
|
|
218
242
|
headers = DEFAULT_HEADERS
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
243
|
+
.merge(@extra_headers)
|
|
244
|
+
.merge("X-Correlation-Id" => SecureRandom.uuid)
|
|
245
|
+
.merge(extra_request_headers)
|
|
222
246
|
|
|
223
247
|
resp = @http_client.post(url, body: body, headers: headers)
|
|
224
248
|
|
|
@@ -245,11 +269,11 @@ module Pinot
|
|
|
245
269
|
broker_response
|
|
246
270
|
rescue *RETRYABLE_HTTP_ERRORS, *RETRYABLE_ERRORS => e
|
|
247
271
|
if attempts < max_attempts
|
|
248
|
-
sleep_ms = (@retry_interval_ms || 200) * (2
|
|
272
|
+
sleep_ms = (@retry_interval_ms || 200) * (2**(attempts - 1))
|
|
249
273
|
sleep(sleep_ms / 1000.0)
|
|
250
274
|
retry
|
|
251
275
|
end
|
|
252
|
-
raise
|
|
276
|
+
raise(Net::ReadTimeout === e || Net::WriteTimeout === e ? QueryTimeoutError.new(e.message) : e)
|
|
253
277
|
end
|
|
254
278
|
end
|
|
255
279
|
|
|
@@ -282,9 +306,7 @@ module Pinot
|
|
|
282
306
|
if request.query_format == "sql"
|
|
283
307
|
parts << "groupByMode=sql;responseFormat=sql"
|
|
284
308
|
parts << "useMultistageEngine=true" if request.use_multistage_engine
|
|
285
|
-
if @timeout_ms && @timeout_ms > 0
|
|
286
|
-
parts << "timeoutMs=#{@timeout_ms}"
|
|
287
|
-
end
|
|
309
|
+
parts << "timeoutMs=#{@timeout_ms}" if @timeout_ms && @timeout_ms > 0
|
|
288
310
|
parts << "timeoutMs=#{request.query_timeout_ms}" if request.query_timeout_ms
|
|
289
311
|
end
|
|
290
312
|
parts.join(";")
|
data/lib/pinot/version.rb
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
require "json"
|
|
2
|
-
require "set"
|
|
3
2
|
require_relative "table_aware_broker_selector"
|
|
4
3
|
require_relative "errors"
|
|
5
4
|
|
|
6
5
|
module Pinot
|
|
7
6
|
class ZookeeperBrokerSelector < TableAwareBrokerSelector
|
|
8
7
|
# ZK path where Pinot stores broker external view
|
|
9
|
-
BROKER_EXTERNAL_VIEW_PATH = "/EXTERNALVIEW/brokerResource"
|
|
8
|
+
BROKER_EXTERNAL_VIEW_PATH = "/EXTERNALVIEW/brokerResource".freeze
|
|
10
9
|
|
|
11
10
|
def initialize(zk_path:, zk_client: nil)
|
|
12
11
|
super()
|
|
@@ -39,9 +38,13 @@ module Pinot
|
|
|
39
38
|
end
|
|
40
39
|
|
|
41
40
|
def setup_watcher
|
|
42
|
-
@zk.register(BROKER_EXTERNAL_VIEW_PATH) do |
|
|
43
|
-
|
|
44
|
-
|
|
41
|
+
@zk.register(BROKER_EXTERNAL_VIEW_PATH) do |_event|
|
|
42
|
+
begin
|
|
43
|
+
fetch_and_update
|
|
44
|
+
rescue StandardError
|
|
45
|
+
nil
|
|
46
|
+
end
|
|
47
|
+
setup_watcher # re-register watch after each trigger
|
|
45
48
|
end
|
|
46
49
|
# Set initial watch
|
|
47
50
|
@zk.exists?(BROKER_EXTERNAL_VIEW_PATH, watch: true)
|
|
@@ -61,12 +64,15 @@ module Pinot
|
|
|
61
64
|
brokers = []
|
|
62
65
|
broker_map.each do |broker_key, state|
|
|
63
66
|
next unless state == "ONLINE"
|
|
67
|
+
|
|
64
68
|
# Broker key format: Broker_<hostname>_<port>
|
|
65
69
|
# Use the last segment as port and the second-to-last as host
|
|
66
70
|
parts = broker_key.split("_")
|
|
67
71
|
next if parts.length < 2
|
|
72
|
+
|
|
68
73
|
port = parts.last
|
|
69
74
|
next unless port =~ /\A\d+\z/
|
|
75
|
+
|
|
70
76
|
host = parts[-2]
|
|
71
77
|
brokers << "#{host}:#{port}"
|
|
72
78
|
end
|
data/lib/pinot.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pinot-client
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.29.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Xiang Fu
|
|
@@ -94,6 +94,34 @@ dependencies:
|
|
|
94
94
|
- - "~>"
|
|
95
95
|
- !ruby/object:Gem::Version
|
|
96
96
|
version: '0.8'
|
|
97
|
+
- !ruby/object:Gem::Dependency
|
|
98
|
+
name: rubocop
|
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
|
100
|
+
requirements:
|
|
101
|
+
- - "~>"
|
|
102
|
+
- !ruby/object:Gem::Version
|
|
103
|
+
version: '1.65'
|
|
104
|
+
type: :development
|
|
105
|
+
prerelease: false
|
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
107
|
+
requirements:
|
|
108
|
+
- - "~>"
|
|
109
|
+
- !ruby/object:Gem::Version
|
|
110
|
+
version: '1.65'
|
|
111
|
+
- !ruby/object:Gem::Dependency
|
|
112
|
+
name: rubocop-rspec
|
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
|
114
|
+
requirements:
|
|
115
|
+
- - "~>"
|
|
116
|
+
- !ruby/object:Gem::Version
|
|
117
|
+
version: '3.0'
|
|
118
|
+
type: :development
|
|
119
|
+
prerelease: false
|
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
121
|
+
requirements:
|
|
122
|
+
- - "~>"
|
|
123
|
+
- !ruby/object:Gem::Version
|
|
124
|
+
version: '3.0'
|
|
97
125
|
description: A Ruby client for Apache Pinot, mirroring the Go client API
|
|
98
126
|
email:
|
|
99
127
|
executables: []
|