pgbus 0.21.4 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +67 -0
- data/README.md +37 -15
- data/Rakefile +15 -2
- data/app/assets/javascripts/pgbus/stream_source_element.js +17 -3
- data/app/assets/javascripts/pgbus/stream_source_element.min.js +4 -4
- data/app/assets/javascripts/pgbus/stream_source_element.min.js.map +1 -1
- data/app/controllers/pgbus/dead_letter_controller.rb +3 -1
- data/app/controllers/pgbus/jobs_controller.rb +2 -1
- data/app/models/pgbus/blocked_execution.rb +3 -2
- data/app/models/pgbus/queue_state.rb +6 -0
- data/app/models/pgbus/uniqueness_key.rb +11 -0
- data/config/locales/da.yml +1 -0
- data/config/locales/de.yml +1 -0
- data/config/locales/en.yml +1 -0
- data/config/locales/es.yml +1 -0
- data/config/locales/fi.yml +1 -0
- data/config/locales/fr.yml +1 -0
- data/config/locales/it.yml +1 -0
- data/config/locales/ja.yml +1 -0
- data/config/locales/nb.yml +1 -0
- data/config/locales/nl.yml +1 -0
- data/config/locales/pt.yml +1 -0
- data/config/locales/sv.yml +1 -0
- data/lib/generators/pgbus/fold_queue_name_case_generator.rb +57 -0
- data/lib/generators/pgbus/templates/fold_queue_name_case.rb.erb +12 -0
- data/lib/generators/pgbus/templates/tune_autovacuum.rb.erb +2 -2
- data/lib/generators/pgbus/templates/tune_fillfactor.rb.erb +1 -1
- data/lib/pgbus/active_job/adapter.rb +3 -2
- data/lib/pgbus/active_job/executor.rb +197 -46
- data/lib/pgbus/active_job/quiet_logs.rb +57 -0
- data/lib/pgbus/autovacuum_tuning.rb +2 -2
- data/lib/pgbus/batch/sweep.rb +2 -0
- data/lib/pgbus/batch.rb +4 -1
- data/lib/pgbus/cli/dlq.rb +2 -1
- data/lib/pgbus/client/async_commit.rb +149 -0
- data/lib/pgbus/client/native_statements.rb +432 -0
- data/lib/pgbus/client.rb +156 -20
- data/lib/pgbus/concurrency/blocked_execution.rb +10 -1
- data/lib/pgbus/configuration.rb +158 -6
- data/lib/pgbus/dead_letter_header.rb +3 -0
- data/lib/pgbus/doctor.rb +55 -2
- data/lib/pgbus/event_bus/handler.rb +2 -0
- data/lib/pgbus/failed_event_recorder.rb +5 -2
- data/lib/pgbus/generators/migration_detector.rb +29 -2
- data/lib/pgbus/instrumentation.rb +16 -1
- data/lib/pgbus/metrics/subscriber.rb +27 -1
- data/lib/pgbus/process/completer.rb +189 -0
- data/lib/pgbus/process/consumer.rb +15 -1
- data/lib/pgbus/process/worker.rb +57 -4
- data/lib/pgbus/queue_name_fold.rb +145 -0
- data/lib/pgbus/queue_name_validator.rb +20 -2
- data/lib/pgbus/recurring/schedule.rb +22 -13
- data/lib/pgbus/serializer.rb +1 -0
- data/lib/pgbus/stat_buffer.rb +67 -26
- data/lib/pgbus/streams/cursor.rb +17 -6
- data/lib/pgbus/streams/envelope.rb +16 -0
- data/lib/pgbus/streams/key.rb +1 -1
- data/lib/pgbus/table_maintenance.rb +1 -1
- data/lib/pgbus/uniqueness.rb +43 -16
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/visibility_heartbeat.rb +16 -12
- data/lib/pgbus/web/data_source/dead_letter.rb +14 -5
- data/lib/pgbus/web/data_source/resend.rb +185 -0
- data/lib/pgbus/web/data_source/resend_slot.rb +139 -0
- data/lib/pgbus/web/data_source.rb +55 -37
- data/lib/pgbus/web/dead_letter_reason.rb +6 -0
- data/lib/pgbus/web/streamer/connection.rb +36 -29
- data/lib/pgbus/web/streamer/falcon_connection.rb +29 -25
- data/lib/pgbus/web/streamer/heartbeat.rb +14 -3
- data/lib/pgbus/web/streamer/instance.rb +8 -5
- data/lib/pgbus/web/streamer/listener.rb +239 -62
- data/lib/pgbus/web/streamer/outbound_pump.rb +6 -4
- data/lib/pgbus/web/streamer/registry.rb +55 -29
- data/lib/pgbus/web/streamer/stream_event_dispatcher.rb +73 -12
- data/lib/tasks/pgbus_queues.rake +27 -0
- data/sig/pgbus/active_job/adapter.rbs +1 -1
- data/sig/pgbus/active_job/quiet_logs.rbs +11 -0
- data/sig/pgbus/client.rbs +7 -5
- data/sig/pgbus/configuration.rbs +12 -0
- data/sig/pgbus/process/completer.rbs +26 -0
- metadata +14 -3
|
@@ -4,18 +4,34 @@ module Pgbus
|
|
|
4
4
|
module Web
|
|
5
5
|
module Streamer
|
|
6
6
|
# Worker-local in-memory registry of SSE connections indexed by stream
|
|
7
|
-
# name and connection id. Thread-safe via a single mutex.
|
|
8
|
-
# snapshots so iterators never hold the lock.
|
|
7
|
+
# name and connection id. Thread-safe via a single mutex.
|
|
9
8
|
#
|
|
10
|
-
#
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
#
|
|
9
|
+
# Reads are copy-free (issue #621): connections_for and each_connection
|
|
10
|
+
# hand out a frozen snapshot that is built on the first read after a
|
|
11
|
+
# membership change and cached until the next one, never mutated in
|
|
12
|
+
# place (the rule docs/concurrency.md sets for memos handed to other
|
|
13
|
+
# threads). Every wake and every heartbeat tick on an unchanged stream
|
|
14
|
+
# gets the same object back: one uncontended lock, no copy. The
|
|
15
|
+
# snapshot is not rebuilt on register / unregister themselves, so a
|
|
16
|
+
# connect storm of M clients does not copy the M-element list M times
|
|
17
|
+
# (an eager copy-on-write registry measured +35 MB of RSS and 80 ms for
|
|
18
|
+
# 2 000 registers, against 0.4 MB and 2.5 ms for this one). Each
|
|
19
|
+
# stream's members are an identity Hash (insertion-ordered), so a
|
|
20
|
+
# register or an unregister is O(1), not a scan of the stream.
|
|
14
21
|
class Registry
|
|
22
|
+
EMPTY = [].freeze
|
|
23
|
+
|
|
15
24
|
def initialize
|
|
16
25
|
@mutex = Mutex.new
|
|
26
|
+
# stream name → identity Hash of its connections (connection => true),
|
|
27
|
+
# mutated under @mutex only.
|
|
17
28
|
@by_stream = {}
|
|
18
29
|
@by_id = {}
|
|
30
|
+
# stream name → the frozen snapshot of @by_stream[name]; an entry is
|
|
31
|
+
# dropped when that stream's membership changes.
|
|
32
|
+
@stream_snapshots = {}
|
|
33
|
+
# The frozen snapshot of every connection, nil after a change.
|
|
34
|
+
@all_snapshot = nil
|
|
19
35
|
end
|
|
20
36
|
|
|
21
37
|
def register(connection)
|
|
@@ -32,8 +48,11 @@ module Pgbus
|
|
|
32
48
|
# return orphaned objects and writes would go nowhere.
|
|
33
49
|
evict_from_stream(existing) if existing
|
|
34
50
|
|
|
51
|
+
# Not already a member: the same object under its id returned
|
|
52
|
+
# above, and a connection's id never changes.
|
|
35
53
|
@by_id[connection.id] = connection
|
|
36
|
-
(@by_stream[connection.stream_name] ||=
|
|
54
|
+
(@by_stream[connection.stream_name] ||= {}.compare_by_identity)[connection] = true
|
|
55
|
+
membership_changed(connection.stream_name)
|
|
37
56
|
end
|
|
38
57
|
end
|
|
39
58
|
|
|
@@ -48,11 +67,8 @@ module Pgbus
|
|
|
48
67
|
existing = @by_id.delete(connection.id)
|
|
49
68
|
return false unless existing
|
|
50
69
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
set.delete(existing)
|
|
54
|
-
@by_stream.delete(existing.stream_name) if set.empty?
|
|
55
|
-
end
|
|
70
|
+
evict_from_stream(existing)
|
|
71
|
+
membership_changed(existing.stream_name)
|
|
56
72
|
true
|
|
57
73
|
end
|
|
58
74
|
end
|
|
@@ -61,25 +77,27 @@ module Pgbus
|
|
|
61
77
|
@mutex.synchronize { @by_id[id] }
|
|
62
78
|
end
|
|
63
79
|
|
|
64
|
-
#
|
|
65
|
-
#
|
|
80
|
+
# The frozen Array of connections on the given stream: the same
|
|
81
|
+
# object until the stream's membership changes.
|
|
66
82
|
def connections_for(stream_name)
|
|
67
83
|
@mutex.synchronize do
|
|
68
|
-
|
|
69
|
-
|
|
84
|
+
# No entry for a stream without members: a wake that lands after
|
|
85
|
+
# the last subscriber left must not leave a slot behind for every
|
|
86
|
+
# per-record stream name (order_42, …) the process ever served.
|
|
87
|
+
members = @by_stream[stream_name]
|
|
88
|
+
next EMPTY unless members
|
|
89
|
+
|
|
90
|
+
@stream_snapshots[stream_name] ||= members.keys.freeze
|
|
70
91
|
end
|
|
71
92
|
end
|
|
72
93
|
|
|
73
|
-
#
|
|
94
|
+
# Frozen list of stream names with at least one registered connection.
|
|
74
95
|
def streams
|
|
75
|
-
@mutex.synchronize { @by_stream.keys.
|
|
96
|
+
@mutex.synchronize { @by_stream.keys.freeze }
|
|
76
97
|
end
|
|
77
98
|
|
|
78
99
|
def empty?(stream_name)
|
|
79
|
-
@mutex.synchronize
|
|
80
|
-
set = @by_stream[stream_name]
|
|
81
|
-
set.nil? || set.empty?
|
|
82
|
-
end
|
|
100
|
+
@mutex.synchronize { !@by_stream.key?(stream_name) }
|
|
83
101
|
end
|
|
84
102
|
|
|
85
103
|
def size
|
|
@@ -87,22 +105,30 @@ module Pgbus
|
|
|
87
105
|
end
|
|
88
106
|
|
|
89
107
|
# Yields every registered connection across all streams. The iteration
|
|
90
|
-
# walks
|
|
91
|
-
# unregister without risk of deadlock or skipped items.
|
|
108
|
+
# walks the current frozen snapshot, so the block may safely call back
|
|
109
|
+
# into register/unregister without risk of deadlock or skipped items.
|
|
92
110
|
def each_connection(&)
|
|
93
|
-
snapshot = @mutex.synchronize { @by_id.values.
|
|
111
|
+
snapshot = @mutex.synchronize { @all_snapshot ||= @by_id.values.freeze }
|
|
94
112
|
snapshot.each(&)
|
|
95
113
|
end
|
|
96
114
|
|
|
97
115
|
private
|
|
98
116
|
|
|
117
|
+
# Must be called while holding @mutex.
|
|
118
|
+
def membership_changed(stream_name)
|
|
119
|
+
@stream_snapshots.delete(stream_name)
|
|
120
|
+
@all_snapshot = nil
|
|
121
|
+
end
|
|
122
|
+
|
|
99
123
|
# Must be called while holding @mutex.
|
|
100
124
|
def evict_from_stream(connection)
|
|
101
|
-
|
|
102
|
-
|
|
125
|
+
name = connection.stream_name
|
|
126
|
+
members = @by_stream[name]
|
|
127
|
+
return unless members
|
|
103
128
|
|
|
104
|
-
|
|
105
|
-
@by_stream.delete(
|
|
129
|
+
members.delete(connection)
|
|
130
|
+
@by_stream.delete(name) if members.empty?
|
|
131
|
+
@stream_snapshots.delete(name)
|
|
106
132
|
end
|
|
107
133
|
end
|
|
108
134
|
end
|
|
@@ -78,6 +78,7 @@ module Pgbus
|
|
|
78
78
|
end
|
|
79
79
|
|
|
80
80
|
DEFAULT_READ_LIMIT = 500
|
|
81
|
+
NO_PAIRS = [].freeze
|
|
81
82
|
|
|
82
83
|
attr_reader :stream_counter
|
|
83
84
|
|
|
@@ -124,8 +125,10 @@ module Pgbus
|
|
|
124
125
|
@fanout_write_deadline_ms = @config.streams_fanout_write_deadline_ms
|
|
125
126
|
@connect_write_deadline_ms = @config.streams_write_deadline_ms
|
|
126
127
|
@stream_counter = stream_counter || StreamCounter.new
|
|
127
|
-
# stream_name → Array<[connection, Array<Envelope>]
|
|
128
|
-
|
|
128
|
+
# stream_name → Array<[connection, Array<Envelope>]>. No default
|
|
129
|
+
# proc: a read for a stream nobody is connecting to (a wake after
|
|
130
|
+
# the last subscriber left) must not leave an entry behind.
|
|
131
|
+
@in_flight = {}
|
|
129
132
|
# PGMQ full table name (pgbus_<prefix>_<name>) → logical stream
|
|
130
133
|
# name. Populated on connect so handle_wake can translate
|
|
131
134
|
# Listener::WakeMessage#queue_name (a full table name, because
|
|
@@ -265,7 +268,7 @@ module Pgbus
|
|
|
265
268
|
started_at = monotonic_ms
|
|
266
269
|
stream = @full_to_logical[msg.queue_name] || msg.queue_name
|
|
267
270
|
registered = @registry.connections_for(stream)
|
|
268
|
-
in_flight_pairs = @in_flight
|
|
271
|
+
in_flight_pairs = @in_flight.fetch(stream, NO_PAIRS)
|
|
269
272
|
return if registered.empty? && in_flight_pairs.empty?
|
|
270
273
|
|
|
271
274
|
if msg.payload
|
|
@@ -285,6 +288,7 @@ module Pgbus
|
|
|
285
288
|
|
|
286
289
|
envelopes = raw_envelopes.map { |e| unwrap_stream_envelope(e) }
|
|
287
290
|
max_msg_id = envelopes.map(&:msg_id).max
|
|
291
|
+
frames = frames_for(envelopes) unless registered.empty?
|
|
288
292
|
|
|
289
293
|
registered.each do |conn|
|
|
290
294
|
visible = visible_envelopes_for(envelopes, conn)
|
|
@@ -294,9 +298,9 @@ module Pgbus
|
|
|
294
298
|
# Post even when `visible` is empty, with batch max, so an
|
|
295
299
|
# audience-filtered-away batch still advances the cursor past the
|
|
296
300
|
# hidden window (issue #321 B2). The pump raises on ephemerals.
|
|
297
|
-
@pump.post(conn, visible, max_msg_id, deadline_ms: @fanout_write_deadline_ms)
|
|
301
|
+
@pump.post(conn, visible, max_msg_id, deadline_ms: @fanout_write_deadline_ms, frames: frames)
|
|
298
302
|
else
|
|
299
|
-
safe_enqueue(conn, visible)
|
|
303
|
+
safe_enqueue(conn, visible, frames: frames)
|
|
300
304
|
advance_scanned_cursor(conn, max_msg_id)
|
|
301
305
|
end
|
|
302
306
|
end
|
|
@@ -336,8 +340,9 @@ module Pgbus
|
|
|
336
340
|
event: normalize_sse_event(parsed["event"])
|
|
337
341
|
)
|
|
338
342
|
|
|
343
|
+
frames = frames_for([envelope]) unless registered.empty?
|
|
339
344
|
registered.each do |conn|
|
|
340
|
-
safe_enqueue(conn, visible_envelopes_for([envelope], conn))
|
|
345
|
+
safe_enqueue(conn, visible_envelopes_for([envelope], conn), frames: frames)
|
|
341
346
|
end
|
|
342
347
|
in_flight_pairs.each do |(conn, buffer)|
|
|
343
348
|
buffer.concat(visible_envelopes_for([envelope], conn))
|
|
@@ -367,13 +372,46 @@ module Pgbus
|
|
|
367
372
|
# prefix: pgmq.q_<prefixed>.INSERT. Registry and the in-flight
|
|
368
373
|
# buffer use the logical name. The Dispatcher is the single
|
|
369
374
|
# translator between the two naming worlds.
|
|
375
|
+
#
|
|
376
|
+
# Only a stream's first subscriber waits on the listener (issue
|
|
377
|
+
# #630). remove_listening is issued in exactly one place,
|
|
378
|
+
# cleanup_stream_if_unused, and only when the registry holds no
|
|
379
|
+
# connection on the stream. So a registered connection, dead or
|
|
380
|
+
# alive, proves the stream's last ensure_listening returned after
|
|
381
|
+
# its last remove_listening was issued. The listener runs commands
|
|
382
|
+
# in order, and a LISTEN that raised is acked only once the reconnect
|
|
383
|
+
# that follows has re-LISTENed it, so the LISTEN is active. Two
|
|
384
|
+
# exceptions. A listener thread on its way out (at stop, or after an
|
|
385
|
+
# unexpected error) releases every waiting caller with nothing
|
|
386
|
+
# LISTENed. And a first connect whose ack timed out (a stuck or
|
|
387
|
+
# reconnecting listener) leaves its LISTEN queued, ahead of any
|
|
388
|
+
# later UNLISTEN, to run when the listener recovers; until then the
|
|
389
|
+
# fast-path connects share the window that first connect already
|
|
390
|
+
# had, and a durable stream's next wake re-reads from the min
|
|
391
|
+
# cursor. A listener reconnect re-LISTENs its whole set; a hub
|
|
392
|
+
# failover replays FailoverListener's subscriptions, which the
|
|
393
|
+
# first-connect / last-disconnect calls keep exact. An empty stream
|
|
394
|
+
# always takes the slow path: that covers a connect that died
|
|
395
|
+
# mid-replay and unlistened without ever registering.
|
|
396
|
+
# @full_to_logical is set here and deleted only by that same
|
|
397
|
+
# cleanup, so it is present whenever the stream is not empty.
|
|
398
|
+
#
|
|
399
|
+
# The fast path holds while the listener reconnects, on purpose.
|
|
400
|
+
# Waiting on a reconnecting listener would cost every connect the
|
|
401
|
+
# full ack budget (health_check_ms + 1 s) for as long as the
|
|
402
|
+
# database stays unreachable: the storm this rule removes. A
|
|
403
|
+
# connect in that window has the same exposure as every connection
|
|
404
|
+
# already registered: a durable stream's next wake after the
|
|
405
|
+
# re-LISTEN re-reads from the min cursor.
|
|
370
406
|
full_name = notify_queue_name_for(stream)
|
|
371
|
-
@
|
|
372
|
-
|
|
407
|
+
if @registry.empty?(stream)
|
|
408
|
+
@full_to_logical[full_name] = stream
|
|
409
|
+
@listener.ensure_listening(full_name)
|
|
410
|
+
end
|
|
373
411
|
|
|
374
412
|
# Step 2: install the in-flight buffer BEFORE any read.
|
|
375
413
|
buffer = []
|
|
376
|
-
@in_flight[stream] << [connection, buffer]
|
|
414
|
+
(@in_flight[stream] ||= []) << [connection, buffer]
|
|
377
415
|
|
|
378
416
|
# Step 3: read the archive for anything published before this
|
|
379
417
|
# connect landed, and write to the connection.
|
|
@@ -481,7 +519,7 @@ module Pgbus
|
|
|
481
519
|
# 2. @in_flight[stream] (cleared by remove_in_flight already)
|
|
482
520
|
# 3. Listener's @listening_to set + the PG LISTEN itself
|
|
483
521
|
def cleanup_stream_if_unused(stream)
|
|
484
|
-
return unless @registry.empty?(stream) &&
|
|
522
|
+
return unless @registry.empty?(stream) && !@in_flight.key?(stream)
|
|
485
523
|
|
|
486
524
|
full_name = @full_to_logical.key(stream)
|
|
487
525
|
return unless full_name
|
|
@@ -550,11 +588,32 @@ module Pgbus
|
|
|
550
588
|
# it implicitly. handle_connect passes @connect_write_deadline_ms
|
|
551
589
|
# explicitly so a new client's initial replay keeps the full window
|
|
552
590
|
# (issue #315 item 3).
|
|
553
|
-
|
|
591
|
+
#
|
|
592
|
+
# frames: the wake's prebuilt frames (frames_for); nil on the connect
|
|
593
|
+
# replay and the in-flight drain, which frame per connection.
|
|
594
|
+
def safe_enqueue(connection, envelopes_or_buffer, deadline_ms: @fanout_write_deadline_ms, frames: nil)
|
|
554
595
|
return if connection.dead?
|
|
555
596
|
return if envelopes_or_buffer.empty?
|
|
556
597
|
|
|
557
|
-
connection.enqueue(envelopes_or_buffer, deadline_ms: deadline_ms)
|
|
598
|
+
connection.enqueue(envelopes_or_buffer, deadline_ms: deadline_ms, frames: frames)
|
|
599
|
+
end
|
|
600
|
+
|
|
601
|
+
# One SSE frame per envelope per wake, shared by every connection on
|
|
602
|
+
# the stream (issue #621): the bytes depend on the envelope alone, so
|
|
603
|
+
# framing per connection repeated the same work N times. Keyed by the
|
|
604
|
+
# Integer msg_id, never by the envelope (a StreamEnvelope's #hash walks
|
|
605
|
+
# its payload, which would cost what the framing saves); msg_id is
|
|
606
|
+
# unique within a wake. Frozen, values and Hash, so the frames stay
|
|
607
|
+
# Ractor-shareable across the dispatch seam (issue #527).
|
|
608
|
+
def frames_for(envelopes)
|
|
609
|
+
envelopes.to_h do |envelope|
|
|
610
|
+
frame = Pgbus::Streams::Envelope.message(
|
|
611
|
+
id: envelope.msg_id,
|
|
612
|
+
event: Pgbus::Streams::Envelope.sse_event_for(envelope),
|
|
613
|
+
data: envelope.payload
|
|
614
|
+
)
|
|
615
|
+
[envelope.msg_id, frame.freeze]
|
|
616
|
+
end.freeze
|
|
558
617
|
end
|
|
559
618
|
|
|
560
619
|
def prune_dead(connections)
|
|
@@ -565,6 +624,8 @@ module Pgbus
|
|
|
565
624
|
|
|
566
625
|
def remove_in_flight(stream, connection)
|
|
567
626
|
pairs = @in_flight[stream]
|
|
627
|
+
return unless pairs
|
|
628
|
+
|
|
568
629
|
pairs.reject! { |(conn, _buf)| conn.equal?(connection) }
|
|
569
630
|
@in_flight.delete(stream) if pairs.empty?
|
|
570
631
|
end
|
data/lib/tasks/pgbus_queues.rake
CHANGED
|
@@ -9,6 +9,33 @@ namespace :pgbus do
|
|
|
9
9
|
puts "Done. FIFO indexes created on all queues."
|
|
10
10
|
end
|
|
11
11
|
|
|
12
|
+
# Issue #633: the pgbus:fold_queue_name_case migration runs once; a
|
|
13
|
+
# mixed-case pgmq.meta row an older process recreated after it is folded
|
|
14
|
+
# here (pgbus doctor's "Queue name case" check names them).
|
|
15
|
+
desc "Spell mixed-case queues (pgmq.meta, NOTIFY triggers, pgbus's queue-name columns) in lower case"
|
|
16
|
+
task fold_case: :environment do
|
|
17
|
+
# The fold writes pgmq.meta and pgbus's tables through one connection,
|
|
18
|
+
# the one pgbus's models (and its migrations) use; a database without
|
|
19
|
+
# pgmq.meta there (PGMQ reached through database_url elsewhere) is
|
|
20
|
+
# refused rather than half-folded.
|
|
21
|
+
pool = Pgbus.configuration.connects_to ? Pgbus::BusRecord.connection_pool : ActiveRecord::Base.connection_pool
|
|
22
|
+
names = pool.with_connection do |connection|
|
|
23
|
+
unless connection.select_value("SELECT to_regclass('pgmq.meta')")
|
|
24
|
+
abort "pgmq.meta is not in the database pgbus's models use (#{connection.current_database}); " \
|
|
25
|
+
"run the fold where pgbus's tables and PGMQ live together"
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
prefix = "#{Pgbus.configuration.queue_prefix.to_s.downcase}_"
|
|
29
|
+
found = connection.select_values(
|
|
30
|
+
"SELECT queue_name FROM pgmq.meta WHERE queue_name <> lower(queue_name) " \
|
|
31
|
+
"AND left(lower(queue_name), length(#{connection.quote(prefix)})) = #{connection.quote(prefix)} ORDER BY queue_name"
|
|
32
|
+
)
|
|
33
|
+
connection.transaction { Pgbus::QueueNameFold.run!(connection) }
|
|
34
|
+
found
|
|
35
|
+
end
|
|
36
|
+
puts names.empty? ? "Every queue is already spelled in lower case." : "Folded: #{names.join(", ")}"
|
|
37
|
+
end
|
|
38
|
+
|
|
12
39
|
desc "Create FIFO index on a specific queue (QUEUE=name)"
|
|
13
40
|
task fifo_index: :environment do
|
|
14
41
|
queue = ENV.fetch("QUEUE") do
|
|
@@ -19,7 +19,7 @@ module Pgbus
|
|
|
19
19
|
module Uniqueness
|
|
20
20
|
module ClassMethods
|
|
21
21
|
# Any other keyword raises ArgumentError (lock_ttl: with a pointer to 1.0's removal).
|
|
22
|
-
def ensures_uniqueness: (?strategy: :until_executed | :while_executing, ?key: untyped,
|
|
22
|
+
def ensures_uniqueness: (?strategy: :until_executed | :until_executing | :while_executing, ?key: untyped,
|
|
23
23
|
?on_conflict: :reject | :discard | :log) -> untyped
|
|
24
24
|
def pgbus_uniqueness: () -> Hash[Symbol, untyped]?
|
|
25
25
|
end
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
module Pgbus
|
|
2
|
+
module ActiveJob
|
|
3
|
+
# config.quiet_active_job_logs: a worker process unsubscribes
|
|
4
|
+
# ActiveJob::LogSubscriber from perform_start and perform.
|
|
5
|
+
module QuietLogs
|
|
6
|
+
EVENTS: Array[String]
|
|
7
|
+
|
|
8
|
+
def self.apply!: (?notifier: untyped) -> bool
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
end
|
data/sig/pgbus/client.rbs
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
# public method is missing here.
|
|
4
4
|
module Pgbus
|
|
5
5
|
class Client
|
|
6
|
-
def archive_batch: (untyped queue_name, untyped msg_ids, ?prefixed: untyped) -> untyped
|
|
7
|
-
def archive_message: (untyped queue_name, untyped msg_id, ?prefixed: untyped) -> untyped
|
|
6
|
+
def archive_batch: (untyped queue_name, untyped msg_ids, ?prefixed: untyped, ?job_completion: bool) -> untyped
|
|
7
|
+
def archive_message: (untyped queue_name, untyped msg_id, ?prefixed: untyped, ?job_completion: bool) -> untyped
|
|
8
8
|
def bind_topic: (untyped pattern, untyped queue_name) -> untyped
|
|
9
9
|
def close: () -> untyped
|
|
10
10
|
def config: () -> untyped
|
|
@@ -14,9 +14,9 @@ module Pgbus
|
|
|
14
14
|
def create_fifo_index: (untyped queue_name) -> untyped
|
|
15
15
|
def create_fifo_indexes_all: () -> untyped
|
|
16
16
|
def dead_letter_physical_name: (untyped queue_name, ?prefixed: untyped) -> untyped
|
|
17
|
-
def delete_batch: (untyped queue_name, untyped msg_ids, ?prefixed: untyped) -> untyped
|
|
18
|
-
def delete_message: (untyped queue_name, untyped msg_id, ?prefixed: untyped) -> untyped
|
|
19
|
-
def drop_queue: (untyped queue_name, ?prefixed: untyped) -> untyped
|
|
17
|
+
def delete_batch: (untyped queue_name, untyped msg_ids, ?prefixed: untyped, ?job_completion: bool) -> untyped
|
|
18
|
+
def delete_message: (untyped queue_name, untyped msg_id, ?prefixed: untyped, ?job_completion: bool) -> untyped
|
|
19
|
+
def drop_queue: (untyped queue_name, ?prefixed: untyped) ?{ () -> untyped } -> untyped
|
|
20
20
|
def ensure_all_queues: () -> untyped
|
|
21
21
|
def ensure_dead_letter_queue: (untyped name) -> untyped
|
|
22
22
|
def ensure_fair_index: (untyped queue_name) -> untyped
|
|
@@ -32,7 +32,9 @@ module Pgbus
|
|
|
32
32
|
def message_in_queue?: (untyped queue_name, msg_id: untyped, ?prefixed: untyped) -> bool?
|
|
33
33
|
def message_with_job_id?: (untyped queue_name, job_id: untyped, ?prefixed: untyped) -> bool?
|
|
34
34
|
def metrics: (?untyped queue_name) -> untyped
|
|
35
|
+
def mixed_case_queue_names: () -> Array[String]
|
|
35
36
|
def move_to_dead_letter: (untyped queue_name, untyped message, ?headers: untyped) -> untyped
|
|
37
|
+
def native_ready?: (String full_name) -> bool
|
|
36
38
|
def notify_enabled?: (untyped queue_name) -> bool
|
|
37
39
|
def notify_stream: (untyped stream_name, untyped payload) -> untyped
|
|
38
40
|
def oldest_claimable_ages: (?untyped queue_name) -> untyped
|
data/sig/pgbus/configuration.rbs
CHANGED
|
@@ -30,6 +30,14 @@ module Pgbus
|
|
|
30
30
|
def capsule_named: (untyped name) -> untyped
|
|
31
31
|
def circuit_breaker_enabled: () -> bool
|
|
32
32
|
def circuit_breaker_enabled=: (untyped) -> untyped
|
|
33
|
+
def completed_jobs: () -> (:archive | :delete)
|
|
34
|
+
def completed_jobs=: (untyped value) -> (:archive | :delete)
|
|
35
|
+
def completion_batch_size: () -> Integer
|
|
36
|
+
def completion_batch_size=: (untyped) -> untyped
|
|
37
|
+
def completion_batching: () -> bool
|
|
38
|
+
def completion_batching=: (untyped) -> untyped
|
|
39
|
+
def completion_synchronous_commit: () -> bool
|
|
40
|
+
def completion_synchronous_commit=: (bool value) -> bool
|
|
33
41
|
def connection_guc_mode: () -> Symbol
|
|
34
42
|
def connection_guc_mode=: (untyped mode) -> untyped
|
|
35
43
|
def connection_options: () -> untyped
|
|
@@ -114,6 +122,8 @@ module Pgbus
|
|
|
114
122
|
def metrics_backend=: (untyped) -> untyped
|
|
115
123
|
def metrics_enabled: () -> bool
|
|
116
124
|
def metrics_enabled=: (untyped) -> untyped
|
|
125
|
+
def native_statements: () -> bool
|
|
126
|
+
def native_statements=: (bool value) -> bool
|
|
117
127
|
def notify_presence_change: (untyped stream_name, untyped change) -> untyped
|
|
118
128
|
def outbox_batch_size: () -> Integer
|
|
119
129
|
def outbox_batch_size=: (untyped) -> untyped
|
|
@@ -144,6 +154,8 @@ module Pgbus
|
|
|
144
154
|
def queue_name: (untyped name) -> untyped
|
|
145
155
|
def queue_prefix: () -> String
|
|
146
156
|
def queue_prefix=: (untyped) -> untyped
|
|
157
|
+
def quiet_active_job_logs: () -> bool
|
|
158
|
+
def quiet_active_job_logs=: (bool value) -> bool
|
|
147
159
|
def read_ahead: () -> Integer
|
|
148
160
|
def read_ahead=: (untyped value) -> untyped
|
|
149
161
|
def read_ahead_for: (untyped entry) -> untyped
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
module Pgbus
|
|
2
|
+
module Process
|
|
3
|
+
# Batched completion (issue #606): one per worker when
|
|
4
|
+
# config.completion_batching is on.
|
|
5
|
+
class Completer
|
|
6
|
+
class Stopped < Pgbus::Error
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
class Waiter
|
|
10
|
+
def initialize: () -> void
|
|
11
|
+
def resolve: (Symbol value) -> void
|
|
12
|
+
def fail: (Exception error) -> void
|
|
13
|
+
def value: () -> Symbol
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
BATCH_OPS: Hash[Symbol, Symbol]
|
|
17
|
+
|
|
18
|
+
def initialize: (client: Pgbus::Client, batch_size: Integer, ?completed_jobs: :archive | :delete) -> void
|
|
19
|
+
def complete: (String queue, Integer msg_id) -> (:archived | :already_archived)
|
|
20
|
+
def stop: (?timeout: Numeric?) -> void
|
|
21
|
+
def alive?: () -> bool
|
|
22
|
+
def pending_count: () -> Integer
|
|
23
|
+
def flushing?: () -> bool
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pgbus
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.23.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Mikael Henriksson
|
|
@@ -63,14 +63,14 @@ dependencies:
|
|
|
63
63
|
requirements:
|
|
64
64
|
- - "~>"
|
|
65
65
|
- !ruby/object:Gem::Version
|
|
66
|
-
version: 0.
|
|
66
|
+
version: 0.8.2
|
|
67
67
|
type: :runtime
|
|
68
68
|
prerelease: false
|
|
69
69
|
version_requirements: !ruby/object:Gem::Requirement
|
|
70
70
|
requirements:
|
|
71
71
|
- - "~>"
|
|
72
72
|
- !ruby/object:Gem::Version
|
|
73
|
-
version: 0.
|
|
73
|
+
version: 0.8.2
|
|
74
74
|
- !ruby/object:Gem::Dependency
|
|
75
75
|
name: railties
|
|
76
76
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -302,6 +302,7 @@ files:
|
|
|
302
302
|
- lib/generators/pgbus/add_uniqueness_keys_generator.rb
|
|
303
303
|
- lib/generators/pgbus/add_workflow_signals_generator.rb
|
|
304
304
|
- lib/generators/pgbus/add_workflows_generator.rb
|
|
305
|
+
- lib/generators/pgbus/fold_queue_name_case_generator.rb
|
|
305
306
|
- lib/generators/pgbus/install_generator.rb
|
|
306
307
|
- lib/generators/pgbus/migrate_job_locks_generator.rb
|
|
307
308
|
- lib/generators/pgbus/migration_path.rb
|
|
@@ -325,6 +326,7 @@ files:
|
|
|
325
326
|
- lib/generators/pgbus/templates/add_workflow_signals.rb.erb
|
|
326
327
|
- lib/generators/pgbus/templates/add_workflows.rb.erb
|
|
327
328
|
- lib/generators/pgbus/templates/deploy_tasks.yml.erb
|
|
329
|
+
- lib/generators/pgbus/templates/fold_queue_name_case.rb.erb
|
|
328
330
|
- lib/generators/pgbus/templates/initializer.rb.erb
|
|
329
331
|
- lib/generators/pgbus/templates/migrate_job_locks_to_uniqueness_keys.rb.erb
|
|
330
332
|
- lib/generators/pgbus/templates/migration.rb.erb
|
|
@@ -343,6 +345,7 @@ files:
|
|
|
343
345
|
- lib/pgbus/active_job/current_attributes.rb
|
|
344
346
|
- lib/pgbus/active_job/delivery.rb
|
|
345
347
|
- lib/pgbus/active_job/executor.rb
|
|
348
|
+
- lib/pgbus/active_job/quiet_logs.rb
|
|
346
349
|
- lib/pgbus/autovacuum_tuning.rb
|
|
347
350
|
- lib/pgbus/batch.rb
|
|
348
351
|
- lib/pgbus/batch/builder.rb
|
|
@@ -353,9 +356,11 @@ files:
|
|
|
353
356
|
- lib/pgbus/cli/dashboard.rb
|
|
354
357
|
- lib/pgbus/cli/dlq.rb
|
|
355
358
|
- lib/pgbus/client.rb
|
|
359
|
+
- lib/pgbus/client/async_commit.rb
|
|
356
360
|
- lib/pgbus/client/connection_health.rb
|
|
357
361
|
- lib/pgbus/client/ensure_stream_queue.rb
|
|
358
362
|
- lib/pgbus/client/fair_read.rb
|
|
363
|
+
- lib/pgbus/client/native_statements.rb
|
|
359
364
|
- lib/pgbus/client/notify_lock_retry.rb
|
|
360
365
|
- lib/pgbus/client/notify_stream.rb
|
|
361
366
|
- lib/pgbus/client/read_after.rb
|
|
@@ -463,6 +468,7 @@ files:
|
|
|
463
468
|
- lib/pgbus/pgmq_schema/pgmq_v1.13.0.sql
|
|
464
469
|
- lib/pgbus/pgmq_schema/pgmq_v1.13.1.sql
|
|
465
470
|
- lib/pgbus/process/claim_buffer.rb
|
|
471
|
+
- lib/pgbus/process/completer.rb
|
|
466
472
|
- lib/pgbus/process/consumer.rb
|
|
467
473
|
- lib/pgbus/process/consumer_priority.rb
|
|
468
474
|
- lib/pgbus/process/dispatcher.rb
|
|
@@ -482,6 +488,7 @@ files:
|
|
|
482
488
|
- lib/pgbus/process/wildcard_queue_resolver.rb
|
|
483
489
|
- lib/pgbus/process/worker.rb
|
|
484
490
|
- lib/pgbus/queue_factory.rb
|
|
491
|
+
- lib/pgbus/queue_name_fold.rb
|
|
485
492
|
- lib/pgbus/queue_name_validator.rb
|
|
486
493
|
- lib/pgbus/rate_counter.rb
|
|
487
494
|
- lib/pgbus/recurring/already_recorded.rb
|
|
@@ -532,6 +539,8 @@ files:
|
|
|
532
539
|
- lib/pgbus/web/data_source/list_counts.rb
|
|
533
540
|
- lib/pgbus/web/data_source/process_list.rb
|
|
534
541
|
- lib/pgbus/web/data_source/queue_summary.rb
|
|
542
|
+
- lib/pgbus/web/data_source/resend.rb
|
|
543
|
+
- lib/pgbus/web/data_source/resend_slot.rb
|
|
535
544
|
- lib/pgbus/web/data_source/workflow_actions.rb
|
|
536
545
|
- lib/pgbus/web/data_source/workflow_insights.rb
|
|
537
546
|
- lib/pgbus/web/data_source/workflow_list.rb
|
|
@@ -613,9 +622,11 @@ files:
|
|
|
613
622
|
- lib/tasks/pgbus_streams.rake
|
|
614
623
|
- sig/pgbus.rbs
|
|
615
624
|
- sig/pgbus/active_job/adapter.rbs
|
|
625
|
+
- sig/pgbus/active_job/quiet_logs.rbs
|
|
616
626
|
- sig/pgbus/client.rbs
|
|
617
627
|
- sig/pgbus/configuration.rbs
|
|
618
628
|
- sig/pgbus/configuration/snapshot.rbs
|
|
629
|
+
- sig/pgbus/process/completer.rbs
|
|
619
630
|
- sig/pgbus/streams.rbs
|
|
620
631
|
- sig/pgbus/streams/stream.rbs
|
|
621
632
|
homepage: https://github.com/zoolutions/pgbus
|