pg_pipeline 0.3.3 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +15 -0
- data/lib/pg_pipeline/bounded_queue.rb +8 -6
- data/lib/pg_pipeline/client.rb +55 -37
- data/lib/pg_pipeline/connection_driver.rb +349 -272
- data/lib/pg_pipeline/pool.rb +229 -157
- data/lib/pg_pipeline/prepared_statement.rb +5 -5
- data/lib/pg_pipeline/request.rb +18 -17
- data/lib/pg_pipeline/runtime/notification.rb +2 -1
- data/lib/pg_pipeline/runtime/semaphore.rb +3 -3
- data/lib/pg_pipeline/runtime/task.rb +14 -8
- data/lib/pg_pipeline/runtime.rb +17 -18
- data/lib/pg_pipeline/server_caps.rb +3 -3
- data/lib/pg_pipeline/session.rb +9 -7
- data/lib/pg_pipeline/session_guard.rb +108 -73
- data/lib/pg_pipeline/transaction.rb +33 -34
- data/lib/pg_pipeline/type_maps.rb +37 -22
- data/lib/pg_pipeline/version.rb +1 -1
- metadata +1 -1
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require
|
|
3
|
+
require 'pg'
|
|
4
4
|
|
|
5
|
-
require_relative
|
|
6
|
-
require_relative
|
|
7
|
-
require_relative
|
|
8
|
-
require_relative
|
|
9
|
-
require_relative
|
|
5
|
+
require_relative 'errors'
|
|
6
|
+
require_relative 'runtime'
|
|
7
|
+
require_relative 'bounded_queue'
|
|
8
|
+
require_relative 'server_caps'
|
|
9
|
+
require_relative 'request'
|
|
10
10
|
|
|
11
11
|
module PgPipeline
|
|
12
12
|
class ConnectionDriver
|
|
@@ -47,7 +47,7 @@ module PgPipeline
|
|
|
47
47
|
|
|
48
48
|
@accepting = @running = @draining = @needs_flush = @writer_armed = @request_event_pending = false
|
|
49
49
|
@readable_events = @results_read = @units_completed = @flush_calls =
|
|
50
|
-
|
|
50
|
+
@flush_incomplete = @dispatches = @leaked_watchers = @submitting = 0
|
|
51
51
|
@socket = @owner_task = @reader_task = @writer_task = @dispatching = nil
|
|
52
52
|
@inflight = []
|
|
53
53
|
end
|
|
@@ -66,7 +66,14 @@ module PgPipeline
|
|
|
66
66
|
in_flight: @inflight.size,
|
|
67
67
|
submitting: @submitting,
|
|
68
68
|
needs_flush: @needs_flush,
|
|
69
|
-
fast_sync: @caps.fast_sync
|
|
69
|
+
fast_sync: @caps.fast_sync?
|
|
70
|
+
}.merge(counter_stats)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
private
|
|
74
|
+
|
|
75
|
+
def counter_stats
|
|
76
|
+
{
|
|
70
77
|
readable_events: readable_events,
|
|
71
78
|
results_read: results_read,
|
|
72
79
|
units_completed: units_completed,
|
|
@@ -80,10 +87,12 @@ module PgPipeline
|
|
|
80
87
|
}
|
|
81
88
|
end
|
|
82
89
|
|
|
90
|
+
public
|
|
91
|
+
|
|
83
92
|
def health_check(timeout)
|
|
84
93
|
return true unless available?
|
|
85
94
|
|
|
86
|
-
probe = Request.build(
|
|
95
|
+
probe = Request.build('SELECT 1', nil)
|
|
87
96
|
submit(probe)
|
|
88
97
|
Runtime.with_timeout(timeout) { probe.wait }
|
|
89
98
|
true
|
|
@@ -101,18 +110,18 @@ module PgPipeline
|
|
|
101
110
|
|
|
102
111
|
def graceful_close = DriverOps.graceful_close(self)
|
|
103
112
|
|
|
104
|
-
def abort!(error = ConnectionLostError.new(
|
|
113
|
+
def abort!(error = ConnectionLostError.new('connection aborted'))
|
|
105
114
|
DriverOps.abort!(self, error)
|
|
106
115
|
end
|
|
107
116
|
end
|
|
108
117
|
|
|
109
118
|
module DriverOps
|
|
119
|
+
module_function
|
|
120
|
+
|
|
110
121
|
WATCHER_JOIN_TIMEOUT = 2.0
|
|
111
122
|
OWNER_JOIN_TIMEOUT = 5.0
|
|
112
123
|
WATCHER_POLL_INTERVAL = 0.25
|
|
113
124
|
|
|
114
|
-
module_function
|
|
115
|
-
|
|
116
125
|
def ratio(numerator, denominator)
|
|
117
126
|
return 0.0 if denominator.zero?
|
|
118
127
|
|
|
@@ -122,14 +131,14 @@ module PgPipeline
|
|
|
122
131
|
def warn_flush_coupling_once(caps)
|
|
123
132
|
return if @flush_coupling_warned
|
|
124
133
|
return if caps.fast_sync?
|
|
125
|
-
return if ENV[
|
|
134
|
+
return if ENV['PG_PIPELINE_SILENCE_WARNINGS']
|
|
126
135
|
|
|
127
136
|
@flush_coupling_warned = true
|
|
128
137
|
warn(
|
|
129
138
|
"pg_pipeline: libpq #{caps.libpq_version} couples pipeline Sync with flush " \
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
139
|
+
'(no PQsendPipelineSync). Queries still pipeline and still amortise RTT, but ' \
|
|
140
|
+
'one flush per unit caps local throughput; libpq >= 17 is recommended for ' \
|
|
141
|
+
'maximum throughput. Set PG_PIPELINE_SILENCE_WARNINGS=1 to silence this.'
|
|
133
142
|
)
|
|
134
143
|
end
|
|
135
144
|
|
|
@@ -142,299 +151,366 @@ module PgPipeline
|
|
|
142
151
|
raise ArgumentError, "#{name} must be an integer >= 1"
|
|
143
152
|
end
|
|
144
153
|
|
|
145
|
-
def start(
|
|
146
|
-
raise Error,
|
|
147
|
-
raise Error,
|
|
148
|
-
|
|
149
|
-
d.conn.setnonblocking(true)
|
|
150
|
-
d.conn.enter_pipeline_mode
|
|
151
|
-
d.socket = d.conn.socket_io
|
|
152
|
-
d.accepting = true
|
|
153
|
-
d.running = true
|
|
154
|
+
def start(driver)
|
|
155
|
+
raise Error, 'driver already started' if driver.running
|
|
156
|
+
raise Error, 'driver start requires an active Fiber scheduler' unless Fiber.scheduler
|
|
154
157
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
158
|
+
conn = driver.conn
|
|
159
|
+
conn.setnonblocking(true)
|
|
160
|
+
conn.enter_pipeline_mode
|
|
161
|
+
driver.socket = conn.socket_io
|
|
162
|
+
driver.accepting = true
|
|
163
|
+
driver.running = true
|
|
164
|
+
start_tasks(driver)
|
|
165
|
+
driver
|
|
159
166
|
rescue Exception
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
teardown_watchers(
|
|
163
|
-
join_owner(
|
|
167
|
+
driver.accepting = false
|
|
168
|
+
driver.running = false
|
|
169
|
+
teardown_watchers(driver)
|
|
170
|
+
join_owner(driver)
|
|
164
171
|
raise
|
|
165
172
|
end
|
|
166
173
|
|
|
167
|
-
def
|
|
168
|
-
|
|
169
|
-
|
|
174
|
+
def start_tasks(driver)
|
|
175
|
+
driver.reader_task = Runtime.spawn(name: :reader) { reader_watcher(driver) }
|
|
176
|
+
driver.writer_task = Runtime.spawn(name: :writer) { writer_watcher(driver) }
|
|
177
|
+
driver.owner_task = Runtime.spawn(name: :owner) { owner_loop(driver) }
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def submit(driver, request)
|
|
181
|
+
raise ShutdownError, 'driver is not accepting work' unless driver.accepting
|
|
182
|
+
raise ProtocolError, 'request must be new before submit' unless request.state == :new
|
|
170
183
|
|
|
171
|
-
|
|
184
|
+
driver.submitting += 1
|
|
172
185
|
begin
|
|
173
|
-
|
|
186
|
+
driver.requests.enqueue(request)
|
|
174
187
|
request.queued!
|
|
175
|
-
notify_requests(
|
|
188
|
+
notify_requests(driver)
|
|
176
189
|
ensure
|
|
177
|
-
|
|
178
|
-
d.events.enqueue(:submission_finished) if d.draining && d.running && d.submitting.zero?
|
|
190
|
+
finish_submission(driver)
|
|
179
191
|
end
|
|
180
192
|
|
|
181
193
|
request
|
|
182
194
|
end
|
|
183
195
|
|
|
184
|
-
def
|
|
185
|
-
|
|
196
|
+
def finish_submission(driver)
|
|
197
|
+
submitting = driver.submitting - 1
|
|
198
|
+
driver.submitting = submitting
|
|
199
|
+
return unless driver.draining && driver.running && submitting.zero?
|
|
200
|
+
|
|
201
|
+
driver.events.enqueue(:submission_finished)
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def graceful_close(driver)
|
|
205
|
+
return unless driver.running
|
|
186
206
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
join_owner(
|
|
207
|
+
driver.accepting = false
|
|
208
|
+
driver.requests.close(ShutdownError.new('driver is closing'))
|
|
209
|
+
driver.events.enqueue(:close)
|
|
210
|
+
join_owner(driver)
|
|
191
211
|
nil
|
|
192
212
|
end
|
|
193
213
|
|
|
194
|
-
def abort!(
|
|
195
|
-
return unless
|
|
214
|
+
def abort!(driver, error)
|
|
215
|
+
return unless driver.running
|
|
196
216
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
join_owner(
|
|
217
|
+
driver.accepting = false
|
|
218
|
+
driver.requests.close(not_dispatched_error(error))
|
|
219
|
+
driver.events.enqueue([:abort, error])
|
|
220
|
+
join_owner(driver)
|
|
201
221
|
nil
|
|
202
222
|
end
|
|
203
223
|
|
|
204
|
-
def abort_timed_out_health_probe(
|
|
205
|
-
return true if probe.settled? || !
|
|
206
|
-
return true unless exclusive_health_probe?(
|
|
224
|
+
def abort_timed_out_health_probe(driver, probe)
|
|
225
|
+
return true if probe.settled? || !driver.running
|
|
226
|
+
return true unless exclusive_health_probe?(driver, probe)
|
|
207
227
|
|
|
208
|
-
abort!(
|
|
228
|
+
abort!(driver, ConnectionLostError.new('idle health check timed out'))
|
|
209
229
|
false
|
|
210
230
|
end
|
|
211
231
|
|
|
212
|
-
def exclusive_health_probe?(
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
232
|
+
def exclusive_health_probe?(driver, probe)
|
|
233
|
+
inflight = driver.inflight
|
|
234
|
+
driver.accepting &&
|
|
235
|
+
driver.dispatching.nil? &&
|
|
236
|
+
driver.submitting.zero? &&
|
|
237
|
+
driver.requests.empty? &&
|
|
238
|
+
inflight.length == 1 &&
|
|
239
|
+
inflight.first.equal?(probe)
|
|
219
240
|
end
|
|
220
241
|
|
|
221
|
-
def owner_loop(
|
|
222
|
-
while
|
|
223
|
-
event =
|
|
242
|
+
def owner_loop(driver)
|
|
243
|
+
while driver.running
|
|
244
|
+
event = driver.events.dequeue
|
|
224
245
|
break if event.nil?
|
|
225
246
|
|
|
226
|
-
process_event(
|
|
247
|
+
process_event(driver, event)
|
|
227
248
|
end
|
|
228
249
|
rescue StandardError => e
|
|
229
|
-
fatal_close(
|
|
250
|
+
fatal_close(driver, ConnectionLostError.new("driver crashed: #{e.class}: #{e.message}"))
|
|
230
251
|
ensure
|
|
231
|
-
fatal_close(
|
|
252
|
+
fatal_close(driver, ShutdownError.new('driver owner stopped before shutdown completed')) if driver.running
|
|
232
253
|
end
|
|
233
254
|
|
|
234
|
-
def process_event(
|
|
235
|
-
input_changed =
|
|
255
|
+
def process_event(driver, event)
|
|
256
|
+
input_changed = handle_event(driver, event)
|
|
257
|
+
return if input_changed == :stopped
|
|
258
|
+
|
|
259
|
+
drain_results(driver) if input_changed
|
|
260
|
+
pump_requests(driver) if driver.running
|
|
261
|
+
finish_graceful_close(driver) if driver.running && driver.draining && drained?(driver)
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
def handle_event(driver, event)
|
|
236
265
|
case event
|
|
237
266
|
when :requests
|
|
238
|
-
|
|
267
|
+
driver.request_event_pending = false
|
|
239
268
|
when :submission_finished
|
|
240
269
|
nil
|
|
241
270
|
when :readable
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
read_available(d)
|
|
245
|
-
input_changed = true
|
|
246
|
-
ensure
|
|
247
|
-
d.reader_rearm.enqueue(:rearm) if d.running
|
|
248
|
-
end
|
|
271
|
+
handle_readable_event(driver)
|
|
272
|
+
return true
|
|
249
273
|
when :writable
|
|
250
|
-
|
|
251
|
-
flush_output(d)
|
|
274
|
+
handle_writable_event(driver)
|
|
252
275
|
when :close
|
|
253
|
-
|
|
276
|
+
driver.draining = true
|
|
254
277
|
when Array
|
|
255
278
|
tag, payload = event
|
|
256
|
-
|
|
257
|
-
fatal_close(d, payload)
|
|
258
|
-
return
|
|
259
|
-
end
|
|
279
|
+
return abort_event(driver, tag, payload)
|
|
260
280
|
end
|
|
261
281
|
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
282
|
+
false
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
def handle_writable_event(driver)
|
|
286
|
+
driver.writer_armed = false
|
|
287
|
+
flush_output(driver)
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
def handle_readable_event(driver)
|
|
291
|
+
driver.readable_events += 1
|
|
292
|
+
read_available(driver)
|
|
293
|
+
true
|
|
294
|
+
ensure
|
|
295
|
+
driver.reader_rearm.enqueue(:rearm) if driver.running
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
def abort_event(driver, tag, payload)
|
|
299
|
+
return false unless tag == :abort
|
|
300
|
+
|
|
301
|
+
fatal_close(driver, payload)
|
|
302
|
+
:stopped
|
|
265
303
|
end
|
|
266
304
|
|
|
267
|
-
def notify_requests(
|
|
268
|
-
return if
|
|
269
|
-
return unless
|
|
305
|
+
def notify_requests(driver)
|
|
306
|
+
return if driver.request_event_pending
|
|
307
|
+
return unless driver.running
|
|
270
308
|
|
|
271
|
-
|
|
272
|
-
|
|
309
|
+
driver.request_event_pending = true
|
|
310
|
+
driver.events.enqueue(:requests)
|
|
273
311
|
end
|
|
274
312
|
|
|
275
|
-
def pump_requests(
|
|
313
|
+
def pump_requests(driver)
|
|
276
314
|
dispatched = false
|
|
277
|
-
while
|
|
278
|
-
request =
|
|
315
|
+
while driver.inflight.size < driver.max_in_flight && !driver.requests.empty?
|
|
316
|
+
request = driver.requests.dequeue
|
|
279
317
|
break unless request
|
|
280
318
|
next if request.cancelled?
|
|
281
319
|
|
|
282
|
-
|
|
283
|
-
unless send_unit(d, request)
|
|
284
|
-
d.dispatching = nil
|
|
285
|
-
next
|
|
286
|
-
end
|
|
287
|
-
|
|
288
|
-
request.dispatched!
|
|
289
|
-
d.inflight << request
|
|
290
|
-
d.dispatching = nil
|
|
291
|
-
d.dispatches += 1
|
|
292
|
-
dispatched = true
|
|
320
|
+
dispatched = dispatch_request(driver, request) || dispatched
|
|
293
321
|
end
|
|
294
322
|
|
|
295
|
-
flush_output(
|
|
323
|
+
flush_output(driver) if dispatched && !driver.needs_flush
|
|
296
324
|
end
|
|
297
325
|
|
|
298
|
-
def
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
rescue StandardError => e
|
|
318
|
-
request.reject!(e)
|
|
319
|
-
return false
|
|
320
|
-
end
|
|
326
|
+
def dispatch_request(driver, request)
|
|
327
|
+
driver.dispatching = request
|
|
328
|
+
return false unless send_unit(driver, request)
|
|
329
|
+
|
|
330
|
+
request.dispatched!
|
|
331
|
+
driver.inflight << request
|
|
332
|
+
driver.dispatching = nil
|
|
333
|
+
driver.dispatches += 1
|
|
334
|
+
true
|
|
335
|
+
ensure
|
|
336
|
+
driver.dispatching = nil if request&.settled?
|
|
337
|
+
end
|
|
338
|
+
|
|
339
|
+
def send_unit(driver, request)
|
|
340
|
+
return false unless send_request(driver, request)
|
|
341
|
+
|
|
342
|
+
place_sync(driver)
|
|
343
|
+
true
|
|
344
|
+
end
|
|
321
345
|
|
|
322
|
-
|
|
346
|
+
def send_request(driver, request)
|
|
347
|
+
send_command(driver.conn, request)
|
|
323
348
|
true
|
|
349
|
+
rescue PG::UnableToSend => e
|
|
350
|
+
reject_send_error(driver, request, e, recoverable: reusable_after_send_rejection?(driver))
|
|
351
|
+
rescue PG::Error => e
|
|
352
|
+
reject_send_error(driver, request, e)
|
|
353
|
+
rescue ProtocolError
|
|
354
|
+
raise
|
|
355
|
+
rescue StandardError => e
|
|
356
|
+
request.reject!(e)
|
|
357
|
+
false
|
|
358
|
+
end
|
|
359
|
+
|
|
360
|
+
def place_sync(driver)
|
|
361
|
+
driver.caps.place_sync(driver.conn)
|
|
324
362
|
rescue PG::Error => e
|
|
325
363
|
raise ConnectionLostError, "dispatch Sync failed: #{e.class}: #{e.message}"
|
|
326
364
|
end
|
|
327
365
|
|
|
366
|
+
def reject_send_error(driver, request, error, recoverable: false)
|
|
367
|
+
driver.dispatching = nil
|
|
368
|
+
request.reject!(NotDispatchedError.new("query was rejected before libpq accepted it: #{error.class}: #{error.message}"))
|
|
369
|
+
return false if recoverable
|
|
370
|
+
|
|
371
|
+
raise ConnectionLostError, "dispatch failed: #{error.class}: #{error.message}"
|
|
372
|
+
end
|
|
373
|
+
|
|
328
374
|
def send_command(conn, request)
|
|
329
|
-
|
|
375
|
+
operation = request.operation
|
|
376
|
+
params = request.params
|
|
377
|
+
case operation
|
|
330
378
|
when :query
|
|
331
|
-
conn.send_query_params(request.sql,
|
|
379
|
+
conn.send_query_params(request.sql, params)
|
|
332
380
|
when :prepare
|
|
333
|
-
|
|
334
|
-
conn.send_prepare(request.statement_name, request.sql)
|
|
335
|
-
else
|
|
336
|
-
conn.send_prepare(request.statement_name, request.sql, request.param_types)
|
|
337
|
-
end
|
|
381
|
+
send_prepare(conn, request)
|
|
338
382
|
when :prepared_query
|
|
339
|
-
conn.send_query_prepared(request.statement_name,
|
|
383
|
+
conn.send_query_prepared(request.statement_name, params)
|
|
340
384
|
else
|
|
341
|
-
raise ProtocolError, "unsupported request operation #{
|
|
385
|
+
raise ProtocolError, "unsupported request operation #{operation.inspect}"
|
|
342
386
|
end
|
|
343
387
|
end
|
|
344
388
|
|
|
345
|
-
def
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
389
|
+
def send_prepare(conn, request)
|
|
390
|
+
param_types = request.param_types
|
|
391
|
+
statement_name = request.statement_name
|
|
392
|
+
sql = request.sql
|
|
393
|
+
if param_types.nil?
|
|
394
|
+
conn.send_prepare(statement_name, sql)
|
|
395
|
+
else
|
|
396
|
+
conn.send_prepare(statement_name, sql, param_types)
|
|
397
|
+
end
|
|
398
|
+
end
|
|
399
|
+
|
|
400
|
+
def reusable_after_send_rejection?(driver)
|
|
401
|
+
conn = driver.conn
|
|
402
|
+
!conn.finished? && conn.status == PG::CONNECTION_OK && conn.pipeline_status != PG::PQ_PIPELINE_OFF
|
|
349
403
|
rescue PG::Error
|
|
350
404
|
false
|
|
351
405
|
end
|
|
352
406
|
|
|
353
|
-
def flush_output(
|
|
354
|
-
return unless
|
|
407
|
+
def flush_output(driver)
|
|
408
|
+
return unless driver.running
|
|
355
409
|
|
|
356
|
-
|
|
357
|
-
if
|
|
358
|
-
|
|
410
|
+
driver.flush_calls += 1
|
|
411
|
+
if driver.conn.sync_flush
|
|
412
|
+
driver.needs_flush = false
|
|
359
413
|
else
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
arm_writer(
|
|
414
|
+
driver.needs_flush = true
|
|
415
|
+
driver.flush_incomplete += 1
|
|
416
|
+
arm_writer(driver)
|
|
363
417
|
end
|
|
364
418
|
rescue PG::Error => e
|
|
365
419
|
raise ConnectionLostError, "flush failed: #{e.class}: #{e.message}"
|
|
366
420
|
end
|
|
367
421
|
|
|
368
|
-
def arm_writer(
|
|
369
|
-
return if
|
|
370
|
-
return unless
|
|
422
|
+
def arm_writer(driver)
|
|
423
|
+
return if driver.writer_armed
|
|
424
|
+
return unless driver.running
|
|
371
425
|
|
|
372
|
-
|
|
373
|
-
|
|
426
|
+
driver.writer_armed = true
|
|
427
|
+
driver.writer_commands.enqueue(:wait_writable)
|
|
374
428
|
end
|
|
375
429
|
|
|
376
|
-
def read_available(
|
|
377
|
-
|
|
430
|
+
def read_available(driver)
|
|
431
|
+
driver.conn.consume_input
|
|
378
432
|
rescue PG::Error => e
|
|
379
433
|
raise ConnectionLostError, "read failed: #{e.class}: #{e.message}"
|
|
380
434
|
end
|
|
381
435
|
|
|
382
|
-
def drain_results(
|
|
436
|
+
def drain_results(driver)
|
|
383
437
|
read = 0
|
|
438
|
+
conn = driver.conn
|
|
439
|
+
inflight = driver.inflight
|
|
384
440
|
|
|
385
441
|
begin
|
|
386
|
-
while !
|
|
387
|
-
result =
|
|
442
|
+
while !inflight.empty? && !conn.is_busy
|
|
443
|
+
result = conn.sync_get_result
|
|
388
444
|
read += 1
|
|
389
|
-
request =
|
|
390
|
-
raise ProtocolError,
|
|
445
|
+
request = inflight.first
|
|
446
|
+
raise ProtocolError, 'result without an in-flight request' unless request
|
|
391
447
|
|
|
392
|
-
handle_pipeline_result(
|
|
448
|
+
handle_pipeline_result(driver, request, result)
|
|
393
449
|
end
|
|
394
450
|
ensure
|
|
395
|
-
|
|
451
|
+
driver.results_read += read if read.positive?
|
|
396
452
|
end
|
|
397
453
|
end
|
|
398
454
|
|
|
399
|
-
def handle_pipeline_result(
|
|
455
|
+
def handle_pipeline_result(driver, request, result)
|
|
400
456
|
if result.nil?
|
|
401
457
|
request.query_boundary!
|
|
402
458
|
return
|
|
403
459
|
end
|
|
404
460
|
|
|
405
|
-
|
|
461
|
+
handle_pipeline_status(driver, request, result, result.result_status)
|
|
462
|
+
end
|
|
463
|
+
|
|
464
|
+
def handle_pipeline_status(driver, request, result, status)
|
|
406
465
|
case status
|
|
407
466
|
when PG::PGRES_TUPLES_OK
|
|
408
|
-
accept_tuples_ok(
|
|
467
|
+
accept_tuples_ok(driver, request, result)
|
|
409
468
|
when PG::PGRES_PIPELINE_SYNC
|
|
410
|
-
|
|
411
|
-
complete_front(d, request)
|
|
469
|
+
complete_pipeline_sync(driver, request, result)
|
|
412
470
|
when PG::PGRES_COMMAND_OK, PG::PGRES_EMPTY_QUERY
|
|
413
|
-
|
|
414
|
-
request.accept_result(result)
|
|
471
|
+
accept_command_result(request, result, status)
|
|
415
472
|
when PG::PGRES_FATAL_ERROR
|
|
416
|
-
|
|
417
|
-
request.record_error!(query_error(result), result: result)
|
|
473
|
+
record_fatal_result(request, result, status)
|
|
418
474
|
when PG::PGRES_PIPELINE_ABORTED
|
|
419
|
-
|
|
420
|
-
clear_result(result)
|
|
421
|
-
request.record_error!(PipelineAbortedError.new("pipeline unit aborted"))
|
|
475
|
+
record_aborted_result(request, result, status)
|
|
422
476
|
when PG::PGRES_BAD_RESPONSE
|
|
423
|
-
reject_pipeline_status!(result,
|
|
477
|
+
reject_pipeline_status!(result, 'server response was not understood')
|
|
424
478
|
when PG::PGRES_COPY_IN, PG::PGRES_COPY_OUT, PG::PGRES_COPY_BOTH
|
|
425
|
-
reject_pipeline_status!(result,
|
|
479
|
+
reject_pipeline_status!(result, 'COPY is not supported on the multiplexed pipeline')
|
|
426
480
|
else
|
|
427
481
|
reject_pipeline_status!(result, "unexpected pipeline result status #{status}")
|
|
428
482
|
end
|
|
429
483
|
end
|
|
430
484
|
|
|
431
|
-
def
|
|
485
|
+
def complete_pipeline_sync(driver, request, result)
|
|
486
|
+
clear_result(result)
|
|
487
|
+
complete_front(driver, request)
|
|
488
|
+
end
|
|
489
|
+
|
|
490
|
+
def accept_command_result(request, result, status)
|
|
491
|
+
ensure_before_query_boundary!(request, status)
|
|
492
|
+
request.accept_result(result)
|
|
493
|
+
end
|
|
494
|
+
|
|
495
|
+
def record_fatal_result(request, result, status)
|
|
496
|
+
ensure_before_query_boundary!(request, status)
|
|
497
|
+
request.record_error!(query_error(result), result: result)
|
|
498
|
+
end
|
|
499
|
+
|
|
500
|
+
def record_aborted_result(request, result, status)
|
|
501
|
+
ensure_before_query_boundary!(request, status)
|
|
502
|
+
clear_result(result)
|
|
503
|
+
request.record_error!(PipelineAbortedError.new('pipeline unit aborted'))
|
|
504
|
+
end
|
|
505
|
+
|
|
506
|
+
def accept_tuples_ok(driver, request, result)
|
|
432
507
|
ensure_before_query_boundary!(request, PG::PGRES_TUPLES_OK)
|
|
433
508
|
begin
|
|
434
|
-
apply_type_map(
|
|
509
|
+
apply_type_map(driver, request, result)
|
|
435
510
|
request.accept_result(result)
|
|
436
511
|
rescue StandardError => e
|
|
437
|
-
|
|
512
|
+
detail = e.message
|
|
513
|
+
message = e.is_a?(Error) ? detail : "typed result mapping failed: #{e.class}: #{detail}"
|
|
438
514
|
request.record_error!(QueryError.new(message, cause_result: result), result: result)
|
|
439
515
|
end
|
|
440
516
|
end
|
|
@@ -450,61 +526,62 @@ module PgPipeline
|
|
|
450
526
|
raise ProtocolError, "result status #{status} arrived after query boundary"
|
|
451
527
|
end
|
|
452
528
|
|
|
453
|
-
def complete_front(
|
|
454
|
-
|
|
529
|
+
def complete_front(driver, request)
|
|
530
|
+
inflight = driver.inflight
|
|
531
|
+
raise ProtocolError, 'sync does not match FIFO front' unless request.equal?(inflight.first)
|
|
455
532
|
|
|
456
|
-
|
|
457
|
-
|
|
533
|
+
inflight.shift
|
|
534
|
+
driver.units_completed += 1
|
|
458
535
|
request.finish!
|
|
459
536
|
end
|
|
460
537
|
|
|
461
538
|
def query_error(result)
|
|
462
539
|
message = result.error_message.to_s.strip
|
|
463
|
-
message =
|
|
540
|
+
message = 'query failed' if message.empty?
|
|
464
541
|
QueryError.new(message, cause_result: result)
|
|
465
542
|
end
|
|
466
543
|
|
|
467
|
-
def drained?(
|
|
468
|
-
|
|
544
|
+
def drained?(driver)
|
|
545
|
+
driver.submitting.zero? && driver.requests.empty? && driver.inflight.empty? && !driver.needs_flush && driver.dispatching.nil?
|
|
469
546
|
end
|
|
470
547
|
|
|
471
|
-
def finish_graceful_close(
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
548
|
+
def finish_graceful_close(driver)
|
|
549
|
+
driver.accepting = false
|
|
550
|
+
driver.running = false
|
|
551
|
+
driver.conn.exit_pipeline_mode
|
|
475
552
|
rescue PG::Error => e
|
|
476
|
-
fail_all(
|
|
553
|
+
fail_all(driver, ConnectionLostError.new("failed to exit pipeline mode: #{e.message}"))
|
|
477
554
|
ensure
|
|
478
|
-
teardown_watchers(
|
|
555
|
+
teardown_watchers(driver)
|
|
479
556
|
end
|
|
480
557
|
|
|
481
|
-
def fatal_close(
|
|
482
|
-
return unless
|
|
558
|
+
def fatal_close(driver, error)
|
|
559
|
+
return unless driver.running || driver.accepting
|
|
483
560
|
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
fail_all(
|
|
561
|
+
driver.accepting = false
|
|
562
|
+
driver.running = false
|
|
563
|
+
driver.requests.close(not_dispatched_error(error))
|
|
564
|
+
fail_all(driver, error)
|
|
488
565
|
|
|
489
|
-
teardown_watchers(
|
|
566
|
+
teardown_watchers(driver)
|
|
490
567
|
end
|
|
491
568
|
|
|
492
|
-
def fail_all(
|
|
569
|
+
def fail_all(driver, error)
|
|
570
|
+
dispatching = driver.dispatching
|
|
571
|
+
inflight = driver.inflight
|
|
493
572
|
uncertain = []
|
|
494
|
-
uncertain <<
|
|
495
|
-
uncertain.concat(
|
|
496
|
-
queued =
|
|
573
|
+
uncertain << dispatching if dispatching
|
|
574
|
+
uncertain.concat(inflight)
|
|
575
|
+
queued = driver.requests.drain
|
|
497
576
|
|
|
498
|
-
uncertain.compact.uniq
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
request.reject!(not_dispatched_error(error)) unless request.settled?
|
|
504
|
-
end
|
|
577
|
+
reject_requests(uncertain.compact.uniq) { indeterminate_error(error) }
|
|
578
|
+
reject_requests(queued) { not_dispatched_error(error) }
|
|
579
|
+
driver.dispatching = nil
|
|
580
|
+
inflight.clear
|
|
581
|
+
end
|
|
505
582
|
|
|
506
|
-
|
|
507
|
-
|
|
583
|
+
def reject_requests(requests)
|
|
584
|
+
requests.each { |request| request.reject!(yield) unless request.settled? }
|
|
508
585
|
end
|
|
509
586
|
|
|
510
587
|
def not_dispatched_error(error)
|
|
@@ -518,7 +595,7 @@ module PgPipeline
|
|
|
518
595
|
|
|
519
596
|
IndeterminateResultError.new(
|
|
520
597
|
"#{error.message}; request was dispatched but its Sync was not observed, " \
|
|
521
|
-
|
|
598
|
+
'so execution/commit outcome is indeterminate'
|
|
522
599
|
)
|
|
523
600
|
end
|
|
524
601
|
|
|
@@ -529,76 +606,76 @@ module PgPipeline
|
|
|
529
606
|
WATCHER_POLL_INTERVAL
|
|
530
607
|
end
|
|
531
608
|
|
|
532
|
-
def wait_socket_readable(
|
|
609
|
+
def wait_socket_readable(driver, timeout)
|
|
533
610
|
loop do
|
|
534
|
-
return false unless
|
|
535
|
-
return true if
|
|
611
|
+
return false unless driver.running
|
|
612
|
+
return true if driver.socket.wait_readable(timeout)
|
|
536
613
|
end
|
|
537
614
|
end
|
|
538
615
|
|
|
539
|
-
def wait_socket_writable(
|
|
616
|
+
def wait_socket_writable(driver, timeout)
|
|
540
617
|
loop do
|
|
541
|
-
return false unless
|
|
542
|
-
return true if
|
|
618
|
+
return false unless driver.running
|
|
619
|
+
return true if driver.socket.wait_writable(timeout)
|
|
543
620
|
end
|
|
544
621
|
end
|
|
545
622
|
|
|
546
|
-
def reader_watcher(
|
|
623
|
+
def reader_watcher(driver)
|
|
547
624
|
timeout = watcher_wait_timeout
|
|
548
625
|
|
|
549
|
-
while
|
|
550
|
-
break unless wait_socket_readable(
|
|
626
|
+
while driver.running
|
|
627
|
+
break unless wait_socket_readable(driver, timeout)
|
|
551
628
|
|
|
552
|
-
|
|
553
|
-
command =
|
|
554
|
-
break unless command == :rearm &&
|
|
629
|
+
driver.events.enqueue(:readable)
|
|
630
|
+
command = driver.reader_rearm.dequeue
|
|
631
|
+
break unless command == :rearm && driver.running
|
|
555
632
|
end
|
|
556
633
|
rescue StandardError => e
|
|
557
|
-
if
|
|
558
|
-
|
|
634
|
+
if driver.running
|
|
635
|
+
driver.events.enqueue([:abort, ConnectionLostError.new("reader watcher failed: #{e.class}: #{e.message}")])
|
|
559
636
|
end
|
|
560
637
|
end
|
|
561
638
|
|
|
562
|
-
def writer_watcher(
|
|
639
|
+
def writer_watcher(driver)
|
|
563
640
|
timeout = watcher_wait_timeout
|
|
564
641
|
|
|
565
|
-
while
|
|
566
|
-
command =
|
|
567
|
-
break unless command == :wait_writable &&
|
|
568
|
-
break unless wait_socket_writable(
|
|
642
|
+
while driver.running
|
|
643
|
+
command = driver.writer_commands.dequeue
|
|
644
|
+
break unless command == :wait_writable && driver.running
|
|
645
|
+
break unless wait_socket_writable(driver, timeout)
|
|
569
646
|
|
|
570
|
-
|
|
647
|
+
driver.events.enqueue(:writable) if driver.running
|
|
571
648
|
end
|
|
572
649
|
rescue StandardError => e
|
|
573
|
-
if
|
|
574
|
-
|
|
650
|
+
if driver.running
|
|
651
|
+
driver.events.enqueue([:abort, ConnectionLostError.new("writer watcher failed: #{e.class}: #{e.message}")])
|
|
575
652
|
end
|
|
576
653
|
end
|
|
577
654
|
|
|
578
|
-
def teardown_watchers(
|
|
579
|
-
tasks = release_watchers(
|
|
580
|
-
close_wait_points(
|
|
655
|
+
def teardown_watchers(driver)
|
|
656
|
+
tasks = release_watchers(driver)
|
|
657
|
+
close_wait_points(driver)
|
|
581
658
|
begin
|
|
582
|
-
|
|
659
|
+
driver.socket&.close
|
|
583
660
|
rescue StandardError
|
|
584
661
|
nil
|
|
585
662
|
end
|
|
586
663
|
stop_watchers(tasks)
|
|
587
|
-
join_watchers(
|
|
588
|
-
|
|
589
|
-
safe_close_conn(
|
|
664
|
+
join_watchers(driver, tasks)
|
|
665
|
+
driver.socket = nil
|
|
666
|
+
safe_close_conn(driver)
|
|
590
667
|
nil
|
|
591
668
|
end
|
|
592
669
|
|
|
593
|
-
def release_watchers(
|
|
594
|
-
tasks = [
|
|
595
|
-
|
|
596
|
-
|
|
670
|
+
def release_watchers(driver)
|
|
671
|
+
tasks = [driver.reader_task, driver.writer_task].compact
|
|
672
|
+
driver.reader_task = nil
|
|
673
|
+
driver.writer_task = nil
|
|
597
674
|
tasks
|
|
598
675
|
end
|
|
599
676
|
|
|
600
|
-
def close_wait_points(
|
|
601
|
-
[
|
|
677
|
+
def close_wait_points(driver)
|
|
678
|
+
[driver.reader_rearm, driver.writer_commands, driver.events].each do |queue|
|
|
602
679
|
queue&.close
|
|
603
680
|
rescue StandardError
|
|
604
681
|
nil
|
|
@@ -615,12 +692,12 @@ module PgPipeline
|
|
|
615
692
|
nil
|
|
616
693
|
end
|
|
617
694
|
|
|
618
|
-
def join_watchers(
|
|
695
|
+
def join_watchers(driver, tasks)
|
|
619
696
|
Array(tasks).each do |task|
|
|
620
697
|
task.wait(WATCHER_JOIN_TIMEOUT)
|
|
621
698
|
rescue Runtime::TimeoutError
|
|
622
|
-
|
|
623
|
-
warn_leaked_task(
|
|
699
|
+
driver.leaked_watchers += 1
|
|
700
|
+
warn_leaked_task(driver, task, WATCHER_JOIN_TIMEOUT)
|
|
624
701
|
rescue Runtime::Cancel, StandardError
|
|
625
702
|
nil
|
|
626
703
|
end
|
|
@@ -628,16 +705,16 @@ module PgPipeline
|
|
|
628
705
|
nil
|
|
629
706
|
end
|
|
630
707
|
|
|
631
|
-
def join_owner(
|
|
632
|
-
owner =
|
|
708
|
+
def join_owner(driver)
|
|
709
|
+
owner = driver.owner_task
|
|
633
710
|
return if owner.nil?
|
|
634
711
|
return if Fiber.current.equal?(owner.fiber)
|
|
635
712
|
|
|
636
713
|
begin
|
|
637
714
|
owner.wait(OWNER_JOIN_TIMEOUT)
|
|
638
715
|
rescue Runtime::TimeoutError
|
|
639
|
-
|
|
640
|
-
warn_leaked_task(
|
|
716
|
+
driver.leaked_watchers += 1
|
|
717
|
+
warn_leaked_task(driver, owner, OWNER_JOIN_TIMEOUT)
|
|
641
718
|
rescue Runtime::Cancel, StandardError
|
|
642
719
|
nil
|
|
643
720
|
end
|
|
@@ -645,30 +722,30 @@ module PgPipeline
|
|
|
645
722
|
nil
|
|
646
723
|
end
|
|
647
724
|
|
|
648
|
-
def warn_leaked_task(
|
|
649
|
-
return if ENV[
|
|
725
|
+
def warn_leaked_task(driver, task, timeout)
|
|
726
|
+
return if ENV['PG_PIPELINE_SILENCE_WARNINGS']
|
|
650
727
|
|
|
651
728
|
warn(
|
|
652
729
|
"pg_pipeline: task #{task.name.inspect} did not exit within " \
|
|
653
|
-
"#{timeout}s and has been leaked (total #{
|
|
654
|
-
|
|
730
|
+
"#{timeout}s and has been leaked (total #{driver.leaked_watchers}). " \
|
|
731
|
+
'Closing the socket and, on schedulers without #fiber_interrupt, the ' \
|
|
655
732
|
"#{WATCHER_POLL_INTERVAL}s watcher poll did not release the fiber in time."
|
|
656
733
|
)
|
|
657
734
|
end
|
|
658
735
|
|
|
659
|
-
def safe_close_conn(
|
|
660
|
-
|
|
736
|
+
def safe_close_conn(driver)
|
|
737
|
+
driver.conn.close unless driver.conn.finished?
|
|
661
738
|
rescue StandardError
|
|
662
739
|
nil
|
|
663
740
|
end
|
|
664
741
|
|
|
665
|
-
def apply_type_map(
|
|
742
|
+
def apply_type_map(driver, request, result)
|
|
666
743
|
return unless request.type_map
|
|
667
744
|
|
|
668
|
-
maps =
|
|
669
|
-
raise Error,
|
|
745
|
+
maps = driver.type_maps
|
|
746
|
+
raise Error, 'typed statement has no type-map registry' unless maps
|
|
670
747
|
|
|
671
|
-
maps.apply!(request, result,
|
|
748
|
+
maps.apply!(request, result, driver.conn)
|
|
672
749
|
end
|
|
673
750
|
|
|
674
751
|
def clear_result(result)
|