pgbus 0.17.1 → 0.17.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +42 -0
- data/README.md +83 -13
- data/Rakefile +26 -0
- data/app/assets/javascripts/pgbus/stream_source_element.js +34 -2
- data/app/assets/javascripts/pgbus/stream_source_element.min.js +6 -0
- data/app/assets/javascripts/pgbus/stream_source_element.min.js.map +1 -0
- data/app/controllers/concerns/pgbus/job_listing.rb +37 -0
- data/app/controllers/concerns/pgbus/process_listing.rb +17 -0
- data/app/controllers/pgbus/application_controller.rb +5 -2
- data/app/controllers/pgbus/batches_controller.rb +4 -1
- data/app/controllers/pgbus/dashboard_controller.rb +20 -6
- data/app/controllers/pgbus/dead_letter_controller.rb +27 -3
- data/app/controllers/pgbus/events_controller.rb +52 -3
- data/app/controllers/pgbus/frontends_controller.rb +25 -11
- data/app/controllers/pgbus/insights_controller.rb +19 -1
- data/app/controllers/pgbus/jobs_controller.rb +45 -26
- data/app/controllers/pgbus/locks_controller.rb +7 -2
- data/app/controllers/pgbus/outbox_controller.rb +4 -1
- data/app/controllers/pgbus/processes_controller.rb +3 -1
- data/app/controllers/pgbus/queues_controller.rb +16 -4
- data/app/controllers/pgbus/recurring_tasks_controller.rb +5 -7
- data/app/frontend/pgbus/application.min.js +2 -0
- data/app/frontend/pgbus/application.min.js.map +1 -0
- data/app/frontend/pgbus/modules/charts.js +137 -77
- data/app/frontend/pgbus/modules/charts.min.js +2 -0
- data/app/frontend/pgbus/modules/charts.min.js.map +1 -0
- data/app/frontend/pgbus/style.css +1 -1
- data/app/frontend/pgbus/tailwind.css +18 -0
- data/app/frontend/pgbus/vendor/turbo.min.js +54 -0
- data/app/frontend/pgbus/vendor/turbo.min.js.map +1 -0
- data/app/helpers/pgbus/application_helper.rb +155 -113
- data/app/helpers/pgbus/button_helper.rb +88 -0
- data/app/helpers/pgbus/dashboard_helper.rb +157 -0
- data/app/helpers/pgbus/events_helper.rb +40 -0
- data/app/helpers/pgbus/insights_helper.rb +39 -0
- data/app/helpers/pgbus/processes_helper.rb +79 -0
- data/app/views/layouts/pgbus/application.html.erb +0 -1
- data/app/views/pgbus/batches/_batches_table.html.erb +16 -13
- data/app/views/pgbus/batches/show.html.erb +3 -4
- data/app/views/pgbus/dashboard/_processes_table.html.erb +11 -11
- data/app/views/pgbus/dashboard/_queue_health.html.erb +42 -51
- data/app/views/pgbus/dashboard/_queues_table.html.erb +18 -16
- data/app/views/pgbus/dashboard/_recent_failures.html.erb +16 -12
- data/app/views/pgbus/dashboard/_stats_cards.html.erb +70 -36
- data/app/views/pgbus/dashboard/show.html.erb +1 -1
- data/app/views/pgbus/dead_letter/_error.html.erb +13 -0
- data/app/views/pgbus/dead_letter/_messages_table.html.erb +37 -96
- data/app/views/pgbus/dead_letter/_row.html.erb +75 -0
- data/app/views/pgbus/dead_letter/index.html.erb +6 -9
- data/app/views/pgbus/dead_letter/show.html.erb +37 -21
- data/app/views/pgbus/events/_list.html.erb +55 -0
- data/app/views/pgbus/events/_processed_table.html.erb +45 -0
- data/app/views/pgbus/events/_replay.html.erb +10 -0
- data/app/views/pgbus/events/_row.html.erb +162 -0
- data/app/views/pgbus/events/_tabs.html.erb +13 -0
- data/app/views/pgbus/events/index.html.erb +27 -48
- data/app/views/pgbus/events/show.html.erb +25 -7
- data/app/views/pgbus/insights/_chart_card.html.erb +21 -0
- data/app/views/pgbus/insights/_latency_table.html.erb +20 -0
- data/app/views/pgbus/insights/_status_table.html.erb +19 -0
- data/app/views/pgbus/insights/_throughput_table.html.erb +18 -0
- data/app/views/pgbus/insights/show.html.erb +83 -100
- data/app/views/pgbus/jobs/_list.html.erb +64 -0
- data/app/views/pgbus/jobs/_row.html.erb +107 -0
- data/app/views/pgbus/jobs/_tabs.html.erb +14 -0
- data/app/views/pgbus/jobs/index.html.erb +10 -10
- data/app/views/pgbus/jobs/show.html.erb +18 -20
- data/app/views/pgbus/locks/_concurrency.html.erb +25 -23
- data/app/views/pgbus/locks/_uniqueness.html.erb +20 -22
- data/app/views/pgbus/outbox/index.html.erb +30 -27
- data/app/views/pgbus/processes/_processes_table.html.erb +6 -25
- data/app/views/pgbus/processes/_row.html.erb +48 -0
- data/app/views/pgbus/processes/_status.html.erb +4 -0
- data/app/views/pgbus/queues/_queues_list.html.erb +28 -32
- data/app/views/pgbus/queues/show.html.erb +69 -118
- data/app/views/pgbus/recurring_tasks/_tasks_table.html.erb +75 -76
- data/app/views/pgbus/recurring_tasks/index.html.erb +1 -1
- data/app/views/pgbus/recurring_tasks/show.html.erb +45 -51
- data/app/views/pgbus/shared/_empty_row.html.erb +2 -0
- data/app/views/pgbus/shared/_pager.html.erb +34 -0
- data/config/i18n-tasks.yml +11 -2
- data/config/locales/da.yml +405 -113
- data/config/locales/de.yml +405 -113
- data/config/locales/en.yml +405 -113
- data/config/locales/es.yml +404 -112
- data/config/locales/fi.yml +404 -112
- data/config/locales/fr.yml +406 -114
- data/config/locales/it.yml +405 -113
- data/config/locales/ja.yml +405 -113
- data/config/locales/nb.yml +405 -113
- data/config/locales/nl.yml +405 -113
- data/config/locales/pt.yml +406 -114
- data/config/locales/sv.yml +404 -112
- data/config/routes.rb +1 -1
- data/lib/generators/pgbus/add_presence_generator.rb +2 -1
- data/lib/generators/pgbus/add_presence_seats_generator.rb +45 -0
- data/lib/generators/pgbus/templates/add_presence.rb.erb +32 -7
- data/lib/generators/pgbus/templates/add_presence_seats.rb.erb +38 -0
- data/lib/generators/pgbus/templates/initializer.rb.erb +4 -0
- data/lib/pgbus/active_job/executor.rb +16 -6
- data/lib/pgbus/cli/dlq.rb +32 -5
- data/lib/pgbus/client.rb +4 -2
- data/lib/pgbus/configuration/presence_settings.rb +103 -0
- data/lib/pgbus/configuration.rb +51 -2
- data/lib/pgbus/dead_letter_header.rb +129 -0
- data/lib/pgbus/doctor.rb +8 -4
- data/lib/pgbus/engine.rb +13 -7
- data/lib/pgbus/event_bus/handler.rb +18 -11
- data/lib/pgbus/event_bus/registry.rb +6 -0
- data/lib/pgbus/failed_event_recorder.rb +34 -0
- data/lib/pgbus/generators/migration_detector.rb +9 -1
- data/lib/pgbus/instrumentation.rb +2 -1
- data/lib/pgbus/mcp/base_tool.rb +25 -0
- data/lib/pgbus/mcp/tools/concurrency_tool.rb +23 -4
- data/lib/pgbus/mcp/tools/dlq_detail_tool.rb +4 -2
- data/lib/pgbus/mcp/tools/dlq_tool.rb +10 -4
- data/lib/pgbus/mcp/tools/locks_tool.rb +23 -5
- data/lib/pgbus/mcp/tools/recurring_tool.rb +33 -4
- data/lib/pgbus/outbox/poller.rb +1 -1
- data/lib/pgbus/process/consumer.rb +53 -3
- data/lib/pgbus/process/dispatcher.rb +1 -1
- data/lib/pgbus/process/supervisor.rb +32 -11
- data/lib/pgbus/process/worker.rb +45 -8
- data/lib/pgbus/recurring/scheduler.rb +2 -1
- data/lib/pgbus/streams/presence.rb +317 -81
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source/dashboard_signals.rb +170 -0
- data/lib/pgbus/web/data_source/dead_letter.rb +142 -0
- data/lib/pgbus/web/data_source/event_list.rb +161 -0
- data/lib/pgbus/web/data_source/job_list.rb +323 -0
- data/lib/pgbus/web/data_source/list_counts.rb +80 -0
- data/lib/pgbus/web/data_source/process_list.rb +69 -0
- data/lib/pgbus/web/data_source/queue_summary.rb +122 -0
- data/lib/pgbus/web/data_source.rb +132 -254
- data/lib/pgbus/web/dead_letter_reason.rb +81 -0
- data/lib/pgbus/web/event_state.rb +71 -0
- data/lib/pgbus/web/health_app.rb +7 -3
- data/lib/pgbus/web/health_verdict.rb +78 -0
- data/lib/pgbus/web/insights_charts.rb +84 -0
- data/lib/pgbus/web/job_state.rb +111 -0
- data/lib/pgbus/web/process_activity.rb +97 -0
- data/lib/pgbus/web/process_state.rb +187 -0
- data/lib/pgbus/web/queue_summary.rb +104 -0
- data/lib/pgbus/web/stream_app.rb +18 -6
- data/lib/pgbus/web/streamer/connection.rb +7 -1
- data/lib/pgbus/web/streamer/falcon_connection.rb +7 -1
- data/lib/pgbus/web/streamer/heartbeat.rb +9 -3
- data/lib/pgbus/web/streamer/instance.rb +2 -1
- data/lib/pgbus/web/streamer/presence_tracker.rb +155 -0
- data/lib/pgbus/web/streamer/stream_event_dispatcher.rb +8 -37
- data/lib/pgbus/web/time_format.rb +128 -0
- metadata +55 -4
- data/app/views/pgbus/events/_pending_table.html.erb +0 -169
- data/app/views/pgbus/jobs/_enqueued_table.html.erb +0 -118
- data/app/views/pgbus/jobs/_failed_table.html.erb +0 -76
data/lib/pgbus/process/worker.rb
CHANGED
|
@@ -35,7 +35,8 @@ module Pgbus
|
|
|
35
35
|
rate_counter: nil, wake_signal: nil, stat_buffer: :default,
|
|
36
36
|
notify_listener: nil, notify_retry_at: 0.0,
|
|
37
37
|
notify_retry_backoff: NOTIFY_RETRY_BASE_SECONDS,
|
|
38
|
-
started_at_monotonic: nil, wake_pipe: nil, read_ahead: nil
|
|
38
|
+
started_at_monotonic: nil, wake_pipe: nil, read_ahead: nil,
|
|
39
|
+
recycle_limits: {}, capsule: nil, process: nil)
|
|
39
40
|
@queues = Array(queues)
|
|
40
41
|
@initial_queues = @queues.dup.freeze
|
|
41
42
|
@wildcard = @queues.include?("*")
|
|
@@ -68,6 +69,11 @@ module Pgbus
|
|
|
68
69
|
# in_flight counts every claimed message: buffered and running.
|
|
69
70
|
@in_flight = Concurrent::AtomicFixnum.new(0)
|
|
70
71
|
@read_ahead = read_ahead || config.read_ahead
|
|
72
|
+
# Capsule-level recycle limits (issue #503); each unset key falls back
|
|
73
|
+
# to the global value at check time, so a config change still applies.
|
|
74
|
+
@recycle_limits = recycle_limits
|
|
75
|
+
@capsule = capsule
|
|
76
|
+
@process = process
|
|
71
77
|
@claim_buffer = ClaimBuffer.new(config: config)
|
|
72
78
|
@loop_tick_at = Concurrent::AtomicReference.new(nil)
|
|
73
79
|
@rate_counter = rate_counter || RateCounter.new(:processed, :failed, :dequeued)
|
|
@@ -644,26 +650,33 @@ module Pgbus
|
|
|
644
650
|
end
|
|
645
651
|
|
|
646
652
|
def exceeded_max_jobs?
|
|
647
|
-
|
|
653
|
+
limit = recycle_limit(:max_jobs_per_worker)
|
|
654
|
+
return false unless limit && @jobs_processed.value >= limit
|
|
648
655
|
|
|
649
656
|
Pgbus.logger.info { "[Pgbus] Worker recycling: max_jobs reached (#{@jobs_processed.value})" }
|
|
650
657
|
true
|
|
651
658
|
end
|
|
652
659
|
|
|
653
660
|
def exceeded_max_memory?
|
|
654
|
-
|
|
661
|
+
limit = recycle_limit(:max_memory_mb)
|
|
662
|
+
return false unless limit && current_memory_mb > limit
|
|
655
663
|
|
|
656
|
-
Pgbus.logger.info { "[Pgbus] Worker recycling: memory limit (#{current_memory_mb}MB > #{
|
|
664
|
+
Pgbus.logger.info { "[Pgbus] Worker recycling: memory limit (#{current_memory_mb}MB > #{limit}MB)" }
|
|
657
665
|
true
|
|
658
666
|
end
|
|
659
667
|
|
|
660
668
|
def exceeded_max_lifetime?
|
|
661
|
-
|
|
669
|
+
limit = recycle_limit(:max_worker_lifetime)
|
|
670
|
+
return false unless limit && (monotonic_now - @started_at_monotonic) > limit
|
|
662
671
|
|
|
663
672
|
Pgbus.logger.info { "[Pgbus] Worker recycling: lifetime exceeded" }
|
|
664
673
|
true
|
|
665
674
|
end
|
|
666
675
|
|
|
676
|
+
def recycle_limit(key)
|
|
677
|
+
config.recycle_limit_for(@recycle_limits, key)
|
|
678
|
+
end
|
|
679
|
+
|
|
667
680
|
# Instrumentation payload may report a value up to MEMORY_CHECK_TTL seconds old.
|
|
668
681
|
def current_memory_mb
|
|
669
682
|
MemoryUsage.current_mb
|
|
@@ -816,10 +829,13 @@ module Pgbus
|
|
|
816
829
|
def start_heartbeat
|
|
817
830
|
@heartbeat = Heartbeat.new(
|
|
818
831
|
kind: "worker",
|
|
832
|
+
# capsule/process lead: the dashboard's metadata cell truncates.
|
|
819
833
|
metadata: {
|
|
834
|
+
capsule: @capsule, process: @process,
|
|
820
835
|
queues: queues, threads: threads, pid: ::Process.pid,
|
|
821
|
-
execution_mode: @execution_mode, consumer_priority: @consumer_priority
|
|
822
|
-
|
|
836
|
+
execution_mode: @execution_mode, consumer_priority: @consumer_priority,
|
|
837
|
+
**lifecycle_metadata
|
|
838
|
+
}.compact,
|
|
823
839
|
on_beat: -> { on_heartbeat },
|
|
824
840
|
loop_tick_supplier: -> { @loop_tick_at.get },
|
|
825
841
|
metadata_supplier: -> { throughput_metadata }
|
|
@@ -860,10 +876,31 @@ module Pgbus
|
|
|
860
876
|
"jobs_processed" => @jobs_processed.value,
|
|
861
877
|
"jobs_failed" => @jobs_failed.value,
|
|
862
878
|
"in_flight" => @in_flight.value,
|
|
863
|
-
"buffered" => @claim_buffer.size
|
|
879
|
+
"buffered" => @claim_buffer.size,
|
|
880
|
+
**pool_metadata
|
|
864
881
|
}
|
|
865
882
|
end
|
|
866
883
|
|
|
884
|
+
# Facts the Processes page states (issue #493), fixed for the process's
|
|
885
|
+
# life: when it started and which supervisor forked it (ppid; the page
|
|
886
|
+
# only groups under it when that pid is a supervisor row on this host).
|
|
887
|
+
def lifecycle_metadata
|
|
888
|
+
{ started_at: @started_at.to_f, supervisor_pid: ::Process.ppid }
|
|
889
|
+
end
|
|
890
|
+
|
|
891
|
+
# Read once per beat: one atomic (threads) or one mutex (async) for the
|
|
892
|
+
# pool, MemoryUsage's 5s cache for RSS (a ps subprocess on darwin), and
|
|
893
|
+
# the recycle limits resolved as the recycle check resolves them (a
|
|
894
|
+
# capsule's own when set), so the page follows a config change.
|
|
895
|
+
def pool_metadata
|
|
896
|
+
pool = @pool.metadata
|
|
897
|
+
{
|
|
898
|
+
"capacity" => pool[:capacity], "busy" => pool[:busy], "memory_mb" => current_memory_mb,
|
|
899
|
+
"max_jobs" => recycle_limit(:max_jobs_per_worker), "max_memory_mb" => recycle_limit(:max_memory_mb),
|
|
900
|
+
"max_lifetime" => recycle_limit(:max_worker_lifetime)
|
|
901
|
+
}.compact
|
|
902
|
+
end
|
|
903
|
+
|
|
867
904
|
def shutdown
|
|
868
905
|
Pgbus.logger.info { "[Pgbus] Worker draining thread pool..." }
|
|
869
906
|
stop_wake_source
|
|
@@ -116,7 +116,8 @@ module Pgbus
|
|
|
116
116
|
def start_heartbeat
|
|
117
117
|
@heartbeat = Process::Heartbeat.new(
|
|
118
118
|
kind: "scheduler",
|
|
119
|
-
metadata: { pid: ::Process.pid, tasks: schedule.tasks.size
|
|
119
|
+
metadata: { pid: ::Process.pid, tasks: schedule.tasks.size, started_at: Time.now.to_f,
|
|
120
|
+
supervisor_pid: ::Process.ppid },
|
|
120
121
|
loop_tick_supplier: -> { @loop_tick_at.get }
|
|
121
122
|
)
|
|
122
123
|
@heartbeat.start
|
|
@@ -35,26 +35,230 @@ module Pgbus
|
|
|
35
35
|
# stream pipeline, which means it's transactional, replayable, and
|
|
36
36
|
# filterable just like any other broadcast.
|
|
37
37
|
#
|
|
38
|
-
#
|
|
39
|
-
#
|
|
40
|
-
#
|
|
41
|
-
#
|
|
42
|
-
#
|
|
38
|
+
# Seats (issue #515). A table migrated with `pgbus:add_presence_seats`
|
|
39
|
+
# also keeps one row per SSE connection: `connection_id` (part of the
|
|
40
|
+
# key), `seat` (the client's per-tab id, sent as `?seat=`) and a
|
|
41
|
+
# `state` jsonb for ephemeral per-tab state. Connection-driven presence
|
|
42
|
+
# (`streams_presence_patterns`) writes those rows, so two tabs of one
|
|
43
|
+
# user are two rows and the user stays present until both are gone.
|
|
44
|
+
# Member-keyed calls (no `connection_id:`) keep working on their own
|
|
45
|
+
# row (connection_id IS NULL), exactly as before.
|
|
46
|
+
#
|
|
47
|
+
# presence.join(member_id: "7", connection_id: conn_id, seat: seat)
|
|
48
|
+
# presence.set_state(connection_id: conn_id, state: { selected: 3 })
|
|
49
|
+
# presence.leave(connection_id: conn_id)
|
|
50
|
+
#
|
|
51
|
+
# Seat rows are expired by the streamer heartbeat after their stream's
|
|
52
|
+
# ttl (`Pgbus.configuration.presence_ttl_for`) and every join, leave
|
|
53
|
+
# and expiry is reported through `streams_presence_on_change`.
|
|
54
|
+
#
|
|
55
|
+
# Capability probe for a library building on seats:
|
|
56
|
+
# Pgbus::Streams::Presence.instance_method(:join).parameters
|
|
57
|
+
# .include?([:key, :connection_id]) # the gem has seats
|
|
58
|
+
# Pgbus::Streams::Presence.seats_supported? # the table is migrated
|
|
59
|
+
#
|
|
60
|
+
# NOT included:
|
|
43
61
|
# - Built-in DOM events on the <pgbus-stream-source> element
|
|
44
62
|
# (the application's broadcast block decides the HTML).
|
|
63
|
+
# - Expiry of member-keyed rows (call #sweep! from a cron for those).
|
|
45
64
|
class Presence
|
|
65
|
+
RECORD_COLUMNS = "member_id, metadata, joined_at, last_seen_at"
|
|
66
|
+
SEAT_RECORD_COLUMNS = "member_id, metadata, joined_at, last_seen_at, connection_id, seat, state"
|
|
67
|
+
|
|
68
|
+
# How long a "no seats" answer is trusted before the schema is probed
|
|
69
|
+
# again, so a table migrated under a running process is picked up.
|
|
70
|
+
SCHEMA_RECHECK_SECONDS = 30
|
|
71
|
+
|
|
72
|
+
@schema_mutex = Mutex.new
|
|
73
|
+
@seats_supported = nil
|
|
74
|
+
@probed_at = nil
|
|
75
|
+
|
|
76
|
+
class << self
|
|
77
|
+
# True when pgbus_presence_members has the seat columns (the
|
|
78
|
+
# add_presence_seats migration ran). A true answer is kept for the
|
|
79
|
+
# life of the process; a false one for SCHEMA_RECHECK_SECONDS.
|
|
80
|
+
def seats_supported?
|
|
81
|
+
return @seats_supported if schema_answer_fresh?
|
|
82
|
+
|
|
83
|
+
@schema_mutex.synchronize do
|
|
84
|
+
next @seats_supported if schema_answer_fresh?
|
|
85
|
+
|
|
86
|
+
rows = connection.exec_params(<<~SQL, [])
|
|
87
|
+
SELECT 1 AS present FROM information_schema.columns
|
|
88
|
+
WHERE table_name = 'pgbus_presence_members' AND column_name = 'connection_id'
|
|
89
|
+
AND table_schema = ANY (current_schemas(false))
|
|
90
|
+
SQL
|
|
91
|
+
@probed_at = monotonic_now
|
|
92
|
+
@seats_supported = !rows.to_a.empty?
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# seats_supported?, but a cached "no seats" is probed again first.
|
|
97
|
+
# For the destructive member-keyed statements: on a table migrated
|
|
98
|
+
# under this process their SQL without `connection_id IS NULL`
|
|
99
|
+
# succeeds silently and deletes seat rows.
|
|
100
|
+
def seats_supported_now?
|
|
101
|
+
return true if seats_supported?
|
|
102
|
+
|
|
103
|
+
reset_schema_cache!
|
|
104
|
+
seats_supported?
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def reset_schema_cache!
|
|
108
|
+
@schema_mutex.synchronize do
|
|
109
|
+
@probed_at = nil
|
|
110
|
+
@seats_supported = nil
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Deletes every seat row (connection_id IS NOT NULL) that has not
|
|
115
|
+
# been touched within its stream's ttl and returns the deleted
|
|
116
|
+
# records, each with its "stream_name". Member-keyed rows are left
|
|
117
|
+
# alone. `ttl_for` maps a stream name to seconds; `min_ttl` is the
|
|
118
|
+
# smallest ttl configured, used to narrow the candidates in SQL.
|
|
119
|
+
# The DELETE re-checks the cutoff (a touch between the two
|
|
120
|
+
# statements keeps the row) and claims each row atomically, so
|
|
121
|
+
# concurrent sweepers in other processes never report a row twice.
|
|
122
|
+
def expire_seats!(ttl_for:, min_ttl:)
|
|
123
|
+
return [] unless seats_supported?
|
|
124
|
+
|
|
125
|
+
candidates = connection.exec_params(<<~SQL, [min_ttl])
|
|
126
|
+
SELECT stream_name, connection_id FROM pgbus_presence_members
|
|
127
|
+
WHERE connection_id IS NOT NULL AND last_seen_at < NOW() - make_interval(secs => $1)
|
|
128
|
+
SQL
|
|
129
|
+
candidates.to_a.group_by { |row| row["stream_name"] }.flat_map do |stream_name, rows|
|
|
130
|
+
expire_stream_seats(stream_name, rows.map { |row| row["connection_id"] }, ttl_for.call(stream_name))
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def connection
|
|
135
|
+
# Respect multi-database setups (connects_to). When
|
|
136
|
+
# Pgbus.configuration.connects_to is set, the pgbus tables
|
|
137
|
+
# live in a separate database accessed via Pgbus::BusRecord;
|
|
138
|
+
# the default path uses ActiveRecord::Base. Matches the
|
|
139
|
+
# canonical pattern in Pgbus::Process::QueueLock#connection
|
|
140
|
+
# and Pgbus::Configuration's connection probe.
|
|
141
|
+
unless defined?(::ActiveRecord::Base)
|
|
142
|
+
raise Pgbus::ConfigurationError,
|
|
143
|
+
"Pgbus::Streams::Presence requires ActiveRecord (no AR connection available)"
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
if Pgbus.configuration.connects_to
|
|
147
|
+
Pgbus::BusRecord.connection.raw_connection
|
|
148
|
+
else
|
|
149
|
+
::ActiveRecord::Base.connection.raw_connection
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Reports a seat join/leave to streams_presence_on_change (issue #515).
|
|
154
|
+
def report_change(stream_name, type, record)
|
|
155
|
+
Pgbus.configuration.notify_presence_change(
|
|
156
|
+
stream_name,
|
|
157
|
+
{ type: type, connection_id: record["connection_id"], seat: record["seat"], member_id: record["id"] }
|
|
158
|
+
)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# A Ruby Array of Strings as a Postgres text[] literal.
|
|
162
|
+
def text_array(values)
|
|
163
|
+
"{#{values.map { |v| %("#{v.to_s.gsub(/["\\]/) { |c| "\\#{c}" }}") }.join(",")}}"
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def monotonic_now
|
|
167
|
+
::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
private
|
|
171
|
+
|
|
172
|
+
# Reads each ivar once: a concurrent reset_schema_cache! may nil them.
|
|
173
|
+
def schema_answer_fresh?
|
|
174
|
+
supported = @seats_supported
|
|
175
|
+
probed_at = @probed_at
|
|
176
|
+
return false if supported.nil? || probed_at.nil?
|
|
177
|
+
|
|
178
|
+
supported || monotonic_now - probed_at < SCHEMA_RECHECK_SECONDS
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def expire_stream_seats(stream_name, connection_ids, ttl)
|
|
182
|
+
rows = connection.exec_params(<<~SQL, [stream_name, text_array(connection_ids), ttl])
|
|
183
|
+
DELETE FROM pgbus_presence_members
|
|
184
|
+
WHERE stream_name = $1 AND connection_id = ANY($2::text[])
|
|
185
|
+
AND last_seen_at < NOW() - make_interval(secs => $3)
|
|
186
|
+
RETURNING #{SEAT_RECORD_COLUMNS}
|
|
187
|
+
SQL
|
|
188
|
+
rows.to_a.map do |row|
|
|
189
|
+
record = Record.build(row)
|
|
190
|
+
report_change(stream_name, :leave, record)
|
|
191
|
+
record.merge("stream_name" => stream_name)
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# Turns a pgbus_presence_members row into the public record Hash.
|
|
197
|
+
# Seat columns appear only when the row has them.
|
|
198
|
+
module Record
|
|
199
|
+
def self.build(row)
|
|
200
|
+
record = {
|
|
201
|
+
"id" => row["member_id"],
|
|
202
|
+
"metadata" => parse_json(row["metadata"]),
|
|
203
|
+
"joined_at" => row["joined_at"],
|
|
204
|
+
"last_seen_at" => row["last_seen_at"]
|
|
205
|
+
}
|
|
206
|
+
return record unless row.key?("connection_id")
|
|
207
|
+
|
|
208
|
+
record.merge("connection_id" => row["connection_id"], "seat" => row["seat"],
|
|
209
|
+
"state" => parse_json(row["state"]))
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def self.parse_json(value)
|
|
213
|
+
return value if value.is_a?(Hash)
|
|
214
|
+
return {} if value.nil? || value.empty?
|
|
215
|
+
|
|
216
|
+
JSON.parse(value)
|
|
217
|
+
rescue JSON::ParserError => e
|
|
218
|
+
# Resilience fallback: return an empty hash rather than
|
|
219
|
+
# crashing the presence member list on one corrupt row. Debug
|
|
220
|
+
# log so operators can still find the corruption without
|
|
221
|
+
# impacting production throughput (debug is off by default).
|
|
222
|
+
# Truncate the value so a huge metadata payload doesn't bloat
|
|
223
|
+
# the log line.
|
|
224
|
+
truncated = value.to_s[0, 200]
|
|
225
|
+
Pgbus.logger&.debug do
|
|
226
|
+
"[Pgbus::Streams::Presence] parse_metadata failed (#{e.class}: #{e.message}); raw: #{truncated.inspect}"
|
|
227
|
+
end
|
|
228
|
+
{}
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
|
|
46
232
|
def initialize(stream)
|
|
47
233
|
@stream = stream
|
|
48
234
|
end
|
|
49
235
|
|
|
236
|
+
# True when the table carries seats (see .seats_supported?).
|
|
237
|
+
def seats?
|
|
238
|
+
self.class.seats_supported?
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
# True when the table carries seats, re-probing a cached "no".
|
|
242
|
+
def seats_now?
|
|
243
|
+
self.class.seats_supported_now?
|
|
244
|
+
end
|
|
245
|
+
|
|
50
246
|
# Adds (or refreshes) a member on this stream. Idempotent: calling
|
|
51
247
|
# join twice with the same member_id updates last_seen_at and
|
|
52
248
|
# metadata without creating a duplicate row. Yields the member
|
|
53
249
|
# hash to the optional block so the caller can render an
|
|
54
250
|
# `<turbo-stream action="append">` to broadcast the join.
|
|
55
|
-
|
|
251
|
+
#
|
|
252
|
+
# With `connection_id:` (seat tables only) the row is the
|
|
253
|
+
# connection's: one per SSE connection, carrying the client's `seat`.
|
|
254
|
+
def join(member_id:, metadata: {}, connection_id: nil, seat: nil)
|
|
56
255
|
member_id = member_id.to_s
|
|
57
|
-
record =
|
|
256
|
+
record = if connection_id
|
|
257
|
+
upsert_seat(member_id, metadata, connection_id.to_s, seat&.to_s)
|
|
258
|
+
else
|
|
259
|
+
upsert_member(member_id, metadata)
|
|
260
|
+
end
|
|
261
|
+
self.class.report_change(@stream.name, :join, record) if connection_id
|
|
58
262
|
|
|
59
263
|
if block_given?
|
|
60
264
|
html = yield(record)
|
|
@@ -67,11 +271,19 @@ module Pgbus
|
|
|
67
271
|
# Removes a member. Yields the (former) member hash to the
|
|
68
272
|
# optional block so the caller can broadcast a `remove` action.
|
|
69
273
|
# Returns the deleted record, or nil if the member wasn't present.
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
record =
|
|
274
|
+
# Pass `connection_id:` instead to remove one connection's seat row.
|
|
275
|
+
def leave(member_id: nil, connection_id: nil)
|
|
276
|
+
record = if connection_id && member_id.nil?
|
|
277
|
+
delete_seat(connection_id.to_s)
|
|
278
|
+
elsif member_id && connection_id.nil?
|
|
279
|
+
delete_member(member_id.to_s)
|
|
280
|
+
else
|
|
281
|
+
raise ArgumentError, "leave takes exactly one of member_id: or connection_id:"
|
|
282
|
+
end
|
|
73
283
|
return nil unless record
|
|
74
284
|
|
|
285
|
+
self.class.report_change(@stream.name, :leave, record) if connection_id
|
|
286
|
+
|
|
75
287
|
if block_given?
|
|
76
288
|
html = yield(record)
|
|
77
289
|
@stream.broadcast(html) if html.is_a?(String) && !html.empty?
|
|
@@ -84,32 +296,53 @@ module Pgbus
|
|
|
84
296
|
# Used as a heartbeat mechanism: clients ping touch periodically
|
|
85
297
|
# to stay in the member list, and a sweeper expires anyone who
|
|
86
298
|
# hasn't pinged in N seconds.
|
|
87
|
-
|
|
299
|
+
#
|
|
300
|
+
# With `connection_id:` (a String or an Array of them) it refreshes
|
|
301
|
+
# those connections' seat rows in one statement and returns the ids
|
|
302
|
+
# it found — a missing id means the row was expired while the
|
|
303
|
+
# connection was still alive, and the caller can re-join it.
|
|
304
|
+
def touch(member_id: nil, connection_id: nil)
|
|
305
|
+
raise ArgumentError, "touch takes exactly one of member_id: or connection_id:" unless member_id.nil? ^ connection_id.nil?
|
|
306
|
+
|
|
307
|
+
return touch_seats(Array(connection_id)) if connection_id
|
|
308
|
+
|
|
88
309
|
member_id = member_id.to_s
|
|
89
310
|
connection.exec_params(<<~SQL, [@stream.name, member_id])
|
|
90
311
|
UPDATE pgbus_presence_members
|
|
91
312
|
SET last_seen_at = NOW()
|
|
92
|
-
WHERE stream_name = $1 AND member_id = $2
|
|
313
|
+
WHERE stream_name = $1 AND member_id = $2#{" AND connection_id IS NULL" if seats_now?}
|
|
93
314
|
SQL
|
|
94
315
|
end
|
|
95
316
|
|
|
317
|
+
# Merges `state` into one connection's seat row (keys not given keep
|
|
318
|
+
# their value). Returns the updated record, or nil when the
|
|
319
|
+
# connection has no row on this stream. Does not broadcast: the
|
|
320
|
+
# caller renders whatever its viewers should see.
|
|
321
|
+
def set_state(connection_id:, state:)
|
|
322
|
+
raise ArgumentError, "set_state takes a Hash (got #{state.class})" unless state.is_a?(Hash)
|
|
323
|
+
|
|
324
|
+
rows = connection.exec_params(<<~SQL, [@stream.name, connection_id.to_s, JSON.generate(state)])
|
|
325
|
+
UPDATE pgbus_presence_members
|
|
326
|
+
SET state = state || $3::jsonb, last_seen_at = NOW()
|
|
327
|
+
WHERE stream_name = $1 AND connection_id = $2
|
|
328
|
+
RETURNING #{SEAT_RECORD_COLUMNS}
|
|
329
|
+
SQL
|
|
330
|
+
row = rows.first
|
|
331
|
+
row && Record.build(row)
|
|
332
|
+
end
|
|
333
|
+
|
|
96
334
|
# Returns the current list of members on this stream as an
|
|
97
|
-
# array of hashes (id, metadata, joined_at, last_seen_at).
|
|
335
|
+
# array of hashes (id, metadata, joined_at, last_seen_at). On a seat
|
|
336
|
+
# table each row is one seat and also carries connection_id, seat
|
|
337
|
+
# and state; group by "id" for one entry per member.
|
|
98
338
|
def members
|
|
99
339
|
rows = connection.exec_params(<<~SQL, [@stream.name])
|
|
100
|
-
SELECT
|
|
340
|
+
SELECT #{seats? ? SEAT_RECORD_COLUMNS : RECORD_COLUMNS}
|
|
101
341
|
FROM pgbus_presence_members
|
|
102
342
|
WHERE stream_name = $1
|
|
103
343
|
ORDER BY joined_at
|
|
104
344
|
SQL
|
|
105
|
-
rows.to_a.map
|
|
106
|
-
{
|
|
107
|
-
"id" => row["member_id"],
|
|
108
|
-
"metadata" => parse_metadata(row["metadata"]),
|
|
109
|
-
"joined_at" => row["joined_at"],
|
|
110
|
-
"last_seen_at" => row["last_seen_at"]
|
|
111
|
-
}
|
|
112
|
-
end
|
|
345
|
+
rows.to_a.map { |row| Record.build(row) }
|
|
113
346
|
end
|
|
114
347
|
|
|
115
348
|
# Returns the count of current members. Faster than members.size
|
|
@@ -128,88 +361,91 @@ module Pgbus
|
|
|
128
361
|
#
|
|
129
362
|
# Atomically claims the deletion via DELETE RETURNING so multiple
|
|
130
363
|
# workers running the sweep concurrently won't double-emit leave
|
|
131
|
-
# events.
|
|
364
|
+
# events. Yields each expired record when a block is given.
|
|
132
365
|
def sweep!(older_than:)
|
|
133
366
|
rows = connection.exec_params(<<~SQL, [@stream.name, older_than])
|
|
134
367
|
DELETE FROM pgbus_presence_members
|
|
135
|
-
WHERE stream_name = $1 AND last_seen_at < $2
|
|
136
|
-
RETURNING
|
|
368
|
+
WHERE stream_name = $1 AND last_seen_at < $2#{" AND connection_id IS NULL" if seats_now?}
|
|
369
|
+
RETURNING #{RECORD_COLUMNS}
|
|
137
370
|
SQL
|
|
138
|
-
rows.to_a
|
|
371
|
+
rows = rows.to_a
|
|
372
|
+
rows.each { |row| yield Record.build(row) } if block_given?
|
|
373
|
+
rows.size
|
|
139
374
|
end
|
|
140
375
|
|
|
141
376
|
private
|
|
142
377
|
|
|
143
|
-
|
|
378
|
+
# A process that last probed the pre-seat schema sends the old conflict
|
|
379
|
+
# target, which Postgres rejects once add_presence_seats has replaced
|
|
380
|
+
# the unique index with a partial one. Re-probe and retry once.
|
|
381
|
+
def upsert_member(member_id, metadata, retried: false)
|
|
382
|
+
upsert_member_once(member_id, metadata)
|
|
383
|
+
rescue PG::InvalidColumnReference
|
|
384
|
+
raise if retried
|
|
385
|
+
|
|
386
|
+
self.class.reset_schema_cache!
|
|
387
|
+
upsert_member(member_id, metadata, retried: true)
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
def upsert_member_once(member_id, metadata)
|
|
391
|
+
conflict = seats? ? "(stream_name, member_id) WHERE connection_id IS NULL" : "(stream_name, member_id)"
|
|
144
392
|
rows = connection.exec_params(<<~SQL, [@stream.name, member_id, JSON.generate(metadata)])
|
|
145
393
|
INSERT INTO pgbus_presence_members (stream_name, member_id, metadata, joined_at, last_seen_at)
|
|
146
394
|
VALUES ($1, $2, $3::jsonb, NOW(), NOW())
|
|
147
|
-
ON CONFLICT
|
|
395
|
+
ON CONFLICT #{conflict}
|
|
148
396
|
DO UPDATE SET metadata = EXCLUDED.metadata, last_seen_at = NOW()
|
|
149
|
-
RETURNING
|
|
397
|
+
RETURNING #{RECORD_COLUMNS}
|
|
150
398
|
SQL
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
399
|
+
Record.build(rows.first)
|
|
400
|
+
end
|
|
401
|
+
|
|
402
|
+
def upsert_seat(member_id, metadata, connection_id, seat)
|
|
403
|
+
rows = connection.exec_params(<<~SQL, [@stream.name, member_id, JSON.generate(metadata), connection_id, seat])
|
|
404
|
+
INSERT INTO pgbus_presence_members
|
|
405
|
+
(stream_name, member_id, metadata, connection_id, seat, joined_at, last_seen_at)
|
|
406
|
+
VALUES ($1, $2, $3::jsonb, $4, $5, NOW(), NOW())
|
|
407
|
+
ON CONFLICT (stream_name, connection_id) WHERE connection_id IS NOT NULL
|
|
408
|
+
DO UPDATE SET member_id = EXCLUDED.member_id, metadata = EXCLUDED.metadata,
|
|
409
|
+
seat = EXCLUDED.seat, last_seen_at = NOW()
|
|
410
|
+
RETURNING #{SEAT_RECORD_COLUMNS}
|
|
411
|
+
SQL
|
|
412
|
+
Record.build(rows.first)
|
|
158
413
|
end
|
|
159
414
|
|
|
160
415
|
def delete_member(member_id)
|
|
161
416
|
rows = connection.exec_params(<<~SQL, [@stream.name, member_id])
|
|
162
417
|
DELETE FROM pgbus_presence_members
|
|
163
|
-
WHERE stream_name = $1 AND member_id = $2
|
|
164
|
-
RETURNING
|
|
418
|
+
WHERE stream_name = $1 AND member_id = $2#{" AND connection_id IS NULL" if seats_now?}
|
|
419
|
+
RETURNING #{RECORD_COLUMNS}
|
|
165
420
|
SQL
|
|
166
421
|
row = rows.first
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
{
|
|
170
|
-
"id" => row["member_id"],
|
|
171
|
-
"metadata" => parse_metadata(row["metadata"]),
|
|
172
|
-
"joined_at" => row["joined_at"],
|
|
173
|
-
"last_seen_at" => row["last_seen_at"]
|
|
174
|
-
}
|
|
175
|
-
end
|
|
176
|
-
|
|
177
|
-
def parse_metadata(value)
|
|
178
|
-
return value if value.is_a?(Hash)
|
|
179
|
-
return {} if value.nil? || value.empty?
|
|
180
|
-
|
|
181
|
-
JSON.parse(value)
|
|
182
|
-
rescue JSON::ParserError => e
|
|
183
|
-
# Resilience fallback: return an empty hash rather than
|
|
184
|
-
# crashing the presence member list on one corrupt row. Debug
|
|
185
|
-
# log so operators can still find the corruption without
|
|
186
|
-
# impacting production throughput (debug is off by default).
|
|
187
|
-
# Truncate the value so a huge metadata payload doesn't bloat
|
|
188
|
-
# the log line.
|
|
189
|
-
truncated = value.to_s[0, 200]
|
|
190
|
-
Pgbus.logger&.debug do
|
|
191
|
-
"[Pgbus::Streams::Presence] parse_metadata failed (#{e.class}: #{e.message}); raw: #{truncated.inspect}"
|
|
192
|
-
end
|
|
193
|
-
{}
|
|
422
|
+
row && Record.build(row)
|
|
194
423
|
end
|
|
195
424
|
|
|
196
|
-
def
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
"Pgbus::Streams::Presence requires ActiveRecord (no AR connection available)"
|
|
206
|
-
end
|
|
425
|
+
def delete_seat(connection_id)
|
|
426
|
+
rows = connection.exec_params(<<~SQL, [@stream.name, connection_id])
|
|
427
|
+
DELETE FROM pgbus_presence_members
|
|
428
|
+
WHERE stream_name = $1 AND connection_id = $2
|
|
429
|
+
RETURNING #{SEAT_RECORD_COLUMNS}
|
|
430
|
+
SQL
|
|
431
|
+
row = rows.first
|
|
432
|
+
row && Record.build(row)
|
|
433
|
+
end
|
|
207
434
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
435
|
+
def touch_seats(connection_ids)
|
|
436
|
+
return [] if connection_ids.empty?
|
|
437
|
+
|
|
438
|
+
rows = connection.exec_params(<<~SQL, [@stream.name, self.class.text_array(connection_ids)])
|
|
439
|
+
UPDATE pgbus_presence_members
|
|
440
|
+
SET last_seen_at = NOW()
|
|
441
|
+
WHERE stream_name = $1 AND connection_id = ANY($2::text[])
|
|
442
|
+
RETURNING connection_id
|
|
443
|
+
SQL
|
|
444
|
+
rows.to_a.map { |row| row["connection_id"] }
|
|
445
|
+
end
|
|
446
|
+
|
|
447
|
+
def connection
|
|
448
|
+
self.class.connection
|
|
213
449
|
end
|
|
214
450
|
end
|
|
215
451
|
end
|
data/lib/pgbus/version.rb
CHANGED