pgbus 0.18.2 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +345 -142
- data/README.md +49 -0
- data/app/controllers/pgbus/deploy_tasks_controller.rb +38 -0
- data/app/controllers/pgbus/frontends_controller.rb +10 -8
- data/app/helpers/pgbus/application_helper.rb +4 -3
- data/app/helpers/pgbus/deploy_tasks_helper.rb +77 -0
- data/app/helpers/pgbus/processes_helper.rb +1 -1
- data/app/models/pgbus/deploy_task.rb +104 -0
- data/app/models/pgbus/deploy_task_run.rb +45 -0
- data/app/models/pgbus/job_stat.rb +4 -0
- data/app/models/pgbus/processed_event.rb +3 -1
- data/app/models/pgbus/stream_queue.rb +34 -3
- data/app/models/pgbus/stream_stat.rb +3 -0
- data/app/views/layouts/pgbus/application.html.erb +2 -0
- data/app/views/pgbus/deploy_tasks/_rollover_banner.html.erb +25 -0
- data/app/views/pgbus/deploy_tasks/_tasks_table.html.erb +71 -0
- data/app/views/pgbus/deploy_tasks/index.html.erb +6 -0
- data/app/views/pgbus/deploy_tasks/show.html.erb +134 -0
- data/config/i18n-tasks.yml +2 -0
- data/config/locales/da.yml +97 -0
- data/config/locales/de.yml +97 -0
- data/config/locales/en.yml +97 -0
- data/config/locales/es.yml +97 -0
- data/config/locales/fi.yml +97 -0
- data/config/locales/fr.yml +97 -0
- data/config/locales/it.yml +97 -0
- data/config/locales/ja.yml +97 -0
- data/config/locales/nb.yml +97 -0
- data/config/locales/nl.yml +97 -0
- data/config/locales/pt.yml +97 -0
- data/config/locales/sv.yml +97 -0
- data/config/routes.rb +7 -0
- data/lib/generators/pgbus/add_deploy_tasks_generator.rb +51 -0
- data/lib/generators/pgbus/templates/add_deploy_tasks_tables.rb.erb +43 -0
- data/lib/generators/pgbus/templates/deploy_tasks.yml.erb +42 -0
- data/lib/generators/pgbus/templates/migration.rb.erb +43 -0
- data/lib/pgbus/active_job/current_attributes.rb +3 -1
- data/lib/pgbus/batch.rb +17 -9
- data/lib/pgbus/cli.rb +3 -0
- data/lib/pgbus/client/read_after.rb +10 -1
- data/lib/pgbus/client.rb +54 -9
- data/lib/pgbus/concurrency.rb +3 -0
- data/lib/pgbus/configuration/deploy_settings.rb +46 -0
- data/lib/pgbus/configuration.rb +87 -9
- data/lib/pgbus/current_attributes.rb +3 -1
- data/lib/pgbus/dedup_cache.rb +15 -7
- data/lib/pgbus/deploy/revision.rb +59 -0
- data/lib/pgbus/deploy/rollover.rb +65 -0
- data/lib/pgbus/deploy/runner.rb +259 -0
- data/lib/pgbus/deploy/task.rb +105 -0
- data/lib/pgbus/engine.rb +28 -0
- data/lib/pgbus/event_bus/handler.rb +16 -4
- data/lib/pgbus/execution_pools/async_pool.rb +23 -2
- data/lib/pgbus/execution_pools/thread_pool.rb +22 -0
- data/lib/pgbus/execution_pools.rb +26 -0
- data/lib/pgbus/generators/migration_detector.rb +11 -0
- data/lib/pgbus/instrumentation.rb +3 -0
- data/lib/pgbus/integrations/appsignal/probe.rb +17 -9
- data/lib/pgbus/integrations/appsignal/subscriber.rb +30 -18
- data/lib/pgbus/integrations/appsignal.rb +26 -16
- data/lib/pgbus/mcp/rack_app.rb +1 -1
- data/lib/pgbus/mcp/server.rb +2 -1
- data/lib/pgbus/mcp/tools/deploy_tasks_tool.rb +86 -0
- data/lib/pgbus/mcp.rb +1 -0
- data/lib/pgbus/metrics/subscriber.rb +35 -21
- data/lib/pgbus/outbox/poller.rb +1 -0
- data/lib/pgbus/process/consumer.rb +2 -1
- data/lib/pgbus/process/consumer_priority.rb +1 -1
- data/lib/pgbus/process/dispatcher.rb +18 -1
- data/lib/pgbus/process/heartbeat.rb +7 -2
- data/lib/pgbus/process/lifecycle.rb +5 -5
- data/lib/pgbus/process/notify_listener.rb +32 -18
- data/lib/pgbus/process/supervisor.rb +35 -3
- data/lib/pgbus/process/wake_pipe.rb +4 -1
- data/lib/pgbus/process/worker.rb +5 -2
- data/lib/pgbus/recurring/scheduler.rb +54 -20
- data/lib/pgbus/retry_backoff.rb +3 -0
- data/lib/pgbus/shareable.rb +70 -0
- data/lib/pgbus/streams/broadcastable_override.rb +3 -0
- data/lib/pgbus/streams/presence.rb +6 -7
- data/lib/pgbus/streams.rb +38 -27
- data/lib/pgbus/testing.rb +19 -3
- data/lib/pgbus/uniqueness.rb +3 -0
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/visibility_heartbeat.rb +51 -34
- data/lib/pgbus/web/authentication.rb +19 -2
- data/lib/pgbus/web/data_source/deploy_task_list.rb +114 -0
- data/lib/pgbus/web/data_source/list_counts.rb +10 -7
- data/lib/pgbus/web/data_source.rb +1 -0
- data/lib/pgbus/web/deploy_task_state.rb +51 -0
- data/lib/pgbus/web/health_server.rb +4 -1
- data/lib/pgbus/web/process_activity.rb +1 -1
- data/lib/pgbus/web/streamer/heartbeat.rb +4 -1
- data/lib/pgbus/web/streamer/hub_client.rb +4 -1
- data/lib/pgbus/web/streamer/listener.rb +10 -1
- data/lib/pgbus/web/streamer/master_hub.rb +11 -5
- data/lib/pgbus/web/streamer/master_hub_boot.rb +3 -1
- data/lib/pgbus/web/streamer/outbound_pump.rb +4 -1
- data/lib/pgbus/web/streamer/peer_close_watcher.rb +4 -1
- data/lib/pgbus/web/streamer/stream_event_dispatcher.rb +13 -1
- data/lib/pgbus.rb +92 -30
- data/sig/pgbus/active_job/adapter.rbs +57 -0
- data/sig/pgbus/client.rbs +78 -0
- data/sig/pgbus/configuration.rbs +326 -0
- data/sig/pgbus/streams/stream.rbs +25 -0
- data/sig/pgbus/streams.rbs +22 -0
- data/sig/pgbus.rbs +73 -0
- metadata +27 -1
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Web
|
|
5
|
+
class DataSource
|
|
6
|
+
# The Deploy tasks page (issue #536): declared tasks with their last
|
|
7
|
+
# rollover and manual runs, one task's run history, the rollover the
|
|
8
|
+
# scheduler acts on, and the two actions (Run now, Enable/Disable).
|
|
9
|
+
module DeployTaskList
|
|
10
|
+
RUN_HISTORY = 25
|
|
11
|
+
|
|
12
|
+
# page/per_page nil returns every task (the MCP tool's default). An
|
|
13
|
+
# install without the tables (pgbus:add_deploy_tasks not run) gets the
|
|
14
|
+
# empty state, whose hint names the generator, not an error per refresh.
|
|
15
|
+
def deploy_tasks(page: nil, per_page: nil)
|
|
16
|
+
return [] unless DeployTask.table_exists?
|
|
17
|
+
|
|
18
|
+
scope = DeployTask.order(:key)
|
|
19
|
+
scope = scope.limit(per_page).offset(((page || 1) - 1) * per_page) if per_page
|
|
20
|
+
records = scope.to_a
|
|
21
|
+
runs = DeployTaskRun.latest_for(records.map(&:key)).group_by(&:task_key)
|
|
22
|
+
records.map { |record| deploy_task_row(record, runs.fetch(record.key, [])) }
|
|
23
|
+
rescue StandardError => e
|
|
24
|
+
Pgbus.logger.error { "[Pgbus::Web] Error fetching deploy tasks: #{e.class}: #{e.message}" }
|
|
25
|
+
[]
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def deploy_task(id)
|
|
29
|
+
return nil unless DeployTask.table_exists?
|
|
30
|
+
|
|
31
|
+
record = DeployTask.find_by(id: id)
|
|
32
|
+
return nil unless record
|
|
33
|
+
|
|
34
|
+
# The state reads the same unbounded latest-per-trigger runs as the
|
|
35
|
+
# list, so a run older than the 25 shown still counts.
|
|
36
|
+
runs = DeployTaskRun.for_task(record.key).recent(RUN_HISTORY).to_a
|
|
37
|
+
deploy_task_row(record, DeployTaskRun.latest_for([record.key]).to_a).merge(
|
|
38
|
+
arguments: record.parsed_arguments,
|
|
39
|
+
runs: runs.map { |run| deploy_run_hash(run).merge(queue_name: run.queue_name, msg_id: run.msg_id) }
|
|
40
|
+
)
|
|
41
|
+
rescue StandardError => e
|
|
42
|
+
Pgbus.logger.error { "[Pgbus::Web] Error fetching deploy task #{id}: #{e.class}: #{e.message}" }
|
|
43
|
+
nil
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def toggle_deploy_task(id)
|
|
47
|
+
record = DeployTask.find_by(id: id)
|
|
48
|
+
return nil unless record
|
|
49
|
+
|
|
50
|
+
record.update!(enabled: !record.enabled)
|
|
51
|
+
record.enabled ? :enabled : :disabled
|
|
52
|
+
rescue StandardError => e
|
|
53
|
+
Pgbus.logger.error { "[Pgbus::Web] Error toggling deploy task #{id}: #{e.message}" }
|
|
54
|
+
nil
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Run now: the scheduler's enqueue path with no gate and no claim. It
|
|
58
|
+
# leaves a pending rollover arm alone and records a manual run.
|
|
59
|
+
# :enqueued or :failed, the real outcome.
|
|
60
|
+
def enqueue_deploy_task_now(id)
|
|
61
|
+
record = DeployTask.find_by(id: id)
|
|
62
|
+
return :failed unless record
|
|
63
|
+
# The hidden button is not the boundary: a disabled task is not run.
|
|
64
|
+
return disabled_enqueue(record) unless record.enabled
|
|
65
|
+
|
|
66
|
+
Deploy::Runner.new(config: Pgbus.configuration).enqueue(record.to_task, triggered_by: Deploy::Runner::MANUAL)
|
|
67
|
+
rescue StandardError => e
|
|
68
|
+
Pgbus.logger.error { "[Pgbus::Web] Error enqueuing deploy task #{id}: #{e.class}: #{e.message}" }
|
|
69
|
+
:failed
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# The rollover judged against the newest live scheduler's revision,
|
|
73
|
+
# not this web process's; that one only when no scheduler is live
|
|
74
|
+
# (scheduler: false says so). nil when it cannot be read.
|
|
75
|
+
def deploy_rollover(now: Time.current)
|
|
76
|
+
scheduler, revision = Deploy::Rollover.scheduler_revision(now: now)
|
|
77
|
+
revision = Deploy::Revision.current unless scheduler
|
|
78
|
+
status = Deploy::Rollover.check(revision: revision, now: now)
|
|
79
|
+
{ revision: revision, reason: status.reason, lagging: status.lagging, workers_live: status.workers_live,
|
|
80
|
+
live_by_revision: status.live_by_revision, scheduler: scheduler }
|
|
81
|
+
rescue StandardError => e
|
|
82
|
+
Pgbus.logger.error { "[Pgbus::Web] Error reading the deploy rollover: #{e.class}: #{e.message}" }
|
|
83
|
+
nil
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
private
|
|
87
|
+
|
|
88
|
+
def disabled_enqueue(record)
|
|
89
|
+
Pgbus.logger.warn { "[Pgbus::Web] Deploy task #{record.key} is disabled; Run now refused" }
|
|
90
|
+
:failed
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def deploy_task_row(record, latest_runs)
|
|
94
|
+
by_trigger = latest_runs.index_by(&:triggered_by)
|
|
95
|
+
{
|
|
96
|
+
id: record.id, key: record.key, class_name: record.class_name, command: record.command,
|
|
97
|
+
version: record.version, every_deploy: record.every_deploy, queue_name: record.queue_name,
|
|
98
|
+
priority: record.priority, description: record.description, enabled: record.enabled,
|
|
99
|
+
static: record.static, armed_at: record.armed_at, armed_for: record.armed_for,
|
|
100
|
+
last_rollover_run: deploy_run_hash(by_trigger[Deploy::Runner::ROLLOVER]),
|
|
101
|
+
last_manual_run: deploy_run_hash(by_trigger[Deploy::Runner::MANUAL]),
|
|
102
|
+
created_at: record.created_at, updated_at: record.updated_at
|
|
103
|
+
}
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def deploy_run_hash(run)
|
|
107
|
+
return nil unless run
|
|
108
|
+
|
|
109
|
+
{ run_at: run.run_at, revision: run.revision, version: run.version, triggered_by: run.triggered_by }
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
@@ -30,15 +30,18 @@ module Pgbus
|
|
|
30
30
|
) b ON s.key = b.concurrency_key
|
|
31
31
|
SQL
|
|
32
32
|
|
|
33
|
+
# Model names, resolved when counted (the models autoload lazily):
|
|
34
|
+
# names rather than lambdas, so the constant is plain frozen data.
|
|
33
35
|
LIST_MODELS = {
|
|
34
|
-
batches:
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
36
|
+
batches: :BatchEntry,
|
|
37
|
+
deploy_tasks: :DeployTask,
|
|
38
|
+
job_locks: :UniquenessKey,
|
|
39
|
+
outbox: :OutboxEntry,
|
|
40
|
+
processed_events: :ProcessedEvent,
|
|
41
|
+
recurring_tasks: :RecurringTask
|
|
39
42
|
}.freeze
|
|
40
43
|
|
|
41
|
-
# list – :batches, :job_locks, :concurrency_keys, :outbox, :processed_events or :recurring_tasks.
|
|
44
|
+
# list – :batches, :deploy_tasks, :job_locks, :concurrency_keys, :outbox, :processed_events or :recurring_tasks.
|
|
42
45
|
def list_count(list)
|
|
43
46
|
raise ArgumentError, "unknown list: #{list.inspect}" unless list == :concurrency_keys || LIST_MODELS.key?(list)
|
|
44
47
|
|
|
@@ -62,7 +65,7 @@ module Pgbus
|
|
|
62
65
|
private
|
|
63
66
|
|
|
64
67
|
def bounded_count(list)
|
|
65
|
-
rows = list == :concurrency_keys ? concurrency_keys_probe : LIST_MODELS.fetch(list).
|
|
68
|
+
rows = list == :concurrency_keys ? concurrency_keys_probe : Pgbus.const_get(LIST_MODELS.fetch(list)).limit(COUNT_CAP + 1).count
|
|
66
69
|
Count.new(total: [rows.to_i, COUNT_CAP].min, capped: rows.to_i > COUNT_CAP)
|
|
67
70
|
rescue StandardError => e
|
|
68
71
|
Pgbus.logger.debug { "[Pgbus::Web] Error counting #{list}: #{e.message}" }
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Web
|
|
5
|
+
# The Deploy tasks page's words for one task (issue #536), derived from
|
|
6
|
+
# its row and the rollover the scheduler acts on (DataSource#deploy_rollover),
|
|
7
|
+
# so the page prints the reason the scheduler waits for. Pure.
|
|
8
|
+
#
|
|
9
|
+
# States: disabled, waiting (armed, gate closed, with the gate's reason),
|
|
10
|
+
# due (armed, gate open), ran (not armed, has a rollover run), never. A
|
|
11
|
+
# manual run is appended to any state through manual_args.
|
|
12
|
+
module DeployTaskState
|
|
13
|
+
BADGE_TONES = { "disabled" => :gray, "waiting" => :yellow, "due" => :indigo, "ran" => :green,
|
|
14
|
+
"never" => :gray }.freeze
|
|
15
|
+
|
|
16
|
+
Result = Data.define(:state, :reason_key, :reason_args, :badge_tone, :manual_args)
|
|
17
|
+
|
|
18
|
+
module_function
|
|
19
|
+
|
|
20
|
+
# row – a DataSource#deploy_tasks row. rollover – DataSource#deploy_rollover, or nil when unreadable.
|
|
21
|
+
def present(row, rollover)
|
|
22
|
+
state, key, args = derive(row, rollover)
|
|
23
|
+
manual = row[:last_manual_run]
|
|
24
|
+
Result.new(state: state, reason_key: key, reason_args: args, badge_tone: BADGE_TONES.fetch(state),
|
|
25
|
+
manual_args: manual ? { ago: manual[:run_at] } : nil)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def derive(row, rollover)
|
|
29
|
+
return ["disabled", row[:armed_at] ? "disabled_armed" : "disabled", {}] unless row[:enabled]
|
|
30
|
+
return armed(rollover) if row[:armed_at]
|
|
31
|
+
|
|
32
|
+
run = row[:last_rollover_run]
|
|
33
|
+
return ["never", "never", {}] unless run
|
|
34
|
+
|
|
35
|
+
["ran", "ran", { ago: run[:run_at], revision: run[:revision] }]
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def armed(rollover)
|
|
39
|
+
return ["waiting", "waiting_unknown", {}] unless rollover
|
|
40
|
+
return ["waiting", "waiting_no_scheduler", {}] unless rollover[:scheduler]
|
|
41
|
+
|
|
42
|
+
case rollover[:reason]
|
|
43
|
+
when :ready then ["due", "due", {}]
|
|
44
|
+
when :lagging then ["waiting", "waiting_lagging", { count: rollover[:lagging].size, revision: rollover[:revision] }]
|
|
45
|
+
when :no_workers then ["waiting", "waiting_no_workers", { revision: rollover[:revision] }]
|
|
46
|
+
else ["waiting", "waiting_no_revision", {}]
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
@@ -65,7 +65,10 @@ module Pgbus
|
|
|
65
65
|
|
|
66
66
|
@server = TCPServer.new(@bind, @configured_port)
|
|
67
67
|
@port = @server.addr[1]
|
|
68
|
-
|
|
68
|
+
# The accept loop, owned by this server: started once (the @server
|
|
69
|
+
# check above) by the supervisor, joined (then killed if it hangs) by
|
|
70
|
+
# #stop.
|
|
71
|
+
@thread = Thread.new { accept_loop } # rubocop:disable ThreadSafety/NewThread
|
|
69
72
|
logger.info { "[Pgbus::Web::HealthServer] listening on #{@bind}:#{@port} (/livez, /readyz)" }
|
|
70
73
|
end
|
|
71
74
|
|
|
@@ -11,7 +11,7 @@ module Pgbus
|
|
|
11
11
|
# :uptime and :limit (lifetime), Strings for queue and topic lists.
|
|
12
12
|
module ProcessActivity
|
|
13
13
|
KIND_ORDER = %w[worker consumer dispatcher scheduler outbox_poller].freeze
|
|
14
|
-
RECYCLE_LIMITS = [%w[jobs max_jobs jobs_processed done], %w[memory max_memory_mb memory_mb used]].freeze
|
|
14
|
+
RECYCLE_LIMITS = [%w[jobs max_jobs jobs_processed done].freeze, %w[memory max_memory_mb memory_mb used].freeze].freeze
|
|
15
15
|
|
|
16
16
|
module_function
|
|
17
17
|
|
|
@@ -48,7 +48,10 @@ module Pgbus
|
|
|
48
48
|
return if @running
|
|
49
49
|
|
|
50
50
|
@running = true
|
|
51
|
-
|
|
51
|
+
# The heartbeat's one thread, owned by this object: Instance#start
|
|
52
|
+
# starts it once (under Streamer's @current_mutex); #stop signals
|
|
53
|
+
# and joins it.
|
|
54
|
+
@thread = Thread.new { run_loop } # rubocop:disable ThreadSafety/NewThread
|
|
52
55
|
self
|
|
53
56
|
end
|
|
54
57
|
|
|
@@ -43,7 +43,10 @@ module Pgbus
|
|
|
43
43
|
|
|
44
44
|
def connect
|
|
45
45
|
@sock = UNIXSocket.new(@socket_path)
|
|
46
|
-
|
|
46
|
+
# The client's reader thread, owned by this object: #connect starts
|
|
47
|
+
# it once per socket; it ends when the socket closes, and #stop
|
|
48
|
+
# closes the socket and joins it.
|
|
49
|
+
@reader = Thread.new { reader_loop } # rubocop:disable ThreadSafety/NewThread
|
|
47
50
|
self
|
|
48
51
|
rescue SystemCallError, IOError, ArgumentError, ThreadError => e
|
|
49
52
|
# ArgumentError: a socket path over the platform sun_path limit;
|
|
@@ -35,8 +35,14 @@ module Pgbus
|
|
|
35
35
|
# For a queue named `pgbus_stream_chat` the trigger table is
|
|
36
36
|
# `q_pgbus_stream_chat`, so the channel is `pgmq.q_pgbus_stream_chat.INSERT`.
|
|
37
37
|
class Listener
|
|
38
|
+
# Kept Ractor-shareable (issue #527): the hand-off to the dispatcher
|
|
39
|
+
# is the seam a Ractor dispatcher would pass by reference. The Strings
|
|
40
|
+
# are frozen in place: they are the sender's own (the NOTIFY's, or the
|
|
41
|
+
# hub frame's).
|
|
38
42
|
WakeMessage = Data.define(:queue_name, :payload) do
|
|
39
43
|
def initialize(queue_name:, payload: nil)
|
|
44
|
+
queue_name.freeze
|
|
45
|
+
payload&.freeze
|
|
40
46
|
super
|
|
41
47
|
end
|
|
42
48
|
end
|
|
@@ -101,7 +107,10 @@ module Pgbus
|
|
|
101
107
|
return if @running
|
|
102
108
|
|
|
103
109
|
@running = true
|
|
104
|
-
|
|
110
|
+
# The LISTEN thread, owned by this object: Instance#start starts it
|
|
111
|
+
# once (under Streamer's @current_mutex); #stop clears @running and
|
|
112
|
+
# joins it.
|
|
113
|
+
@thread = Thread.new { run_loop } # rubocop:disable ThreadSafety/NewThread
|
|
105
114
|
self
|
|
106
115
|
end
|
|
107
116
|
|
|
@@ -124,9 +124,12 @@ module Pgbus
|
|
|
124
124
|
File.umask(old_umask)
|
|
125
125
|
end
|
|
126
126
|
File.chmod(0o600, @socket_path)
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
127
|
+
# The hub's three loops, owned by this hub: #start runs once at hub
|
|
128
|
+
# start in the Puma master, under @lifecycle_mutex; #stop signals
|
|
129
|
+
# each loop and joins all three.
|
|
130
|
+
@accept_thread = Thread.new { accept_loop } # rubocop:disable ThreadSafety/NewThread
|
|
131
|
+
@fanout_thread = Thread.new { fanout_loop } # rubocop:disable ThreadSafety/NewThread
|
|
132
|
+
@status_thread = Thread.new { status_loop } # rubocop:disable ThreadSafety/NewThread
|
|
130
133
|
self
|
|
131
134
|
rescue StandardError
|
|
132
135
|
@table_mutex.synchronize { @running = false }
|
|
@@ -201,8 +204,11 @@ module Pgbus
|
|
|
201
204
|
@workers[@next_id] = entry
|
|
202
205
|
@next_id
|
|
203
206
|
end
|
|
204
|
-
|
|
205
|
-
entry
|
|
207
|
+
# One writer and one reader per connected Puma worker, owned by its
|
|
208
|
+
# @workers entry: both end when cleanup_worker closes the entry's
|
|
209
|
+
# socket (a worker hangs up, or #stop cleans up every worker).
|
|
210
|
+
entry[:writer] = Thread.new { writer_loop(id, entry) } # rubocop:disable ThreadSafety/NewThread
|
|
211
|
+
entry[:reader] = Thread.new { reader_loop(id, entry) } # rubocop:disable ThreadSafety/NewThread
|
|
206
212
|
id
|
|
207
213
|
end
|
|
208
214
|
|
|
@@ -50,7 +50,9 @@ module Pgbus
|
|
|
50
50
|
def start
|
|
51
51
|
ENV["PGBUS_STREAMS_HUB_SOCKET"] = @socket_path
|
|
52
52
|
@state_mutex.synchronize { @running = true }
|
|
53
|
-
|
|
53
|
+
# The boot thread, owned by this object: the Puma plugin starts it
|
|
54
|
+
# once in the master; #stop clears @running and joins it.
|
|
55
|
+
@thread = Thread.new { wait_and_start } # rubocop:disable ThreadSafety/NewThread
|
|
54
56
|
self
|
|
55
57
|
end
|
|
56
58
|
|
|
@@ -62,7 +62,10 @@ module Pgbus
|
|
|
62
62
|
|
|
63
63
|
@started = true
|
|
64
64
|
@partitions.each do |partition|
|
|
65
|
-
|
|
65
|
+
# One writer per partition, owned by this pump: Instance#start
|
|
66
|
+
# starts them once (the @started check above); #stop signals each
|
|
67
|
+
# partition and joins every thread.
|
|
68
|
+
@threads << Thread.new { run_worker(partition) } # rubocop:disable ThreadSafety/NewThread
|
|
66
69
|
end
|
|
67
70
|
self
|
|
68
71
|
end
|
|
@@ -57,7 +57,10 @@ module Pgbus
|
|
|
57
57
|
|
|
58
58
|
@wake_reader, @wake_writer = IO.pipe if @wake_reader.closed?
|
|
59
59
|
@running = true
|
|
60
|
-
|
|
60
|
+
# The watcher's one thread, owned by this object: Instance#start
|
|
61
|
+
# starts it once (under Streamer's @current_mutex); #stop wakes it
|
|
62
|
+
# through the pipe and joins it.
|
|
63
|
+
@thread = Thread.new { run_loop } # rubocop:disable ThreadSafety/NewThread
|
|
61
64
|
self
|
|
62
65
|
end
|
|
63
66
|
|
|
@@ -62,8 +62,17 @@ module Pgbus
|
|
|
62
62
|
# nil means the default (turbo-stream); a typed
|
|
63
63
|
# name (e.g. "presence", "reactive") lets clients
|
|
64
64
|
# route without sniffing the HTML (issue #170).
|
|
65
|
+
#
|
|
66
|
+
# Kept Ractor-shareable, like WakeMessage (issue #527): its Strings are
|
|
67
|
+
# frozen in place (the dispatcher's own, from JSON.parse or the read),
|
|
68
|
+
# and `exclude` is normalized to the String it is compared as.
|
|
65
69
|
StreamEnvelope = Data.define(:msg_id, :enqueued_at, :payload, :source, :visible_to, :exclude, :event) do
|
|
66
70
|
def initialize(msg_id:, enqueued_at:, payload:, source:, visible_to: nil, exclude: nil, event: nil)
|
|
71
|
+
enqueued_at&.freeze
|
|
72
|
+
payload&.freeze
|
|
73
|
+
source&.freeze
|
|
74
|
+
event&.freeze
|
|
75
|
+
exclude = exclude&.to_s&.freeze
|
|
67
76
|
super
|
|
68
77
|
end
|
|
69
78
|
end
|
|
@@ -147,7 +156,10 @@ module Pgbus
|
|
|
147
156
|
return if @running
|
|
148
157
|
|
|
149
158
|
@running = true
|
|
150
|
-
|
|
159
|
+
# The dispatcher's one thread, owned by this object: Instance#start
|
|
160
|
+
# starts it once (under Streamer's @current_mutex); #stop sends the
|
|
161
|
+
# sentinel and joins it.
|
|
162
|
+
@thread = Thread.new { run_loop } # rubocop:disable ThreadSafety/NewThread
|
|
151
163
|
self
|
|
152
164
|
end
|
|
153
165
|
|
data/lib/pgbus.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "concurrent/map"
|
|
3
4
|
require "zeitwerk"
|
|
4
5
|
|
|
5
6
|
module Pgbus
|
|
@@ -83,17 +84,37 @@ module Pgbus
|
|
|
83
84
|
end
|
|
84
85
|
|
|
85
86
|
class << self
|
|
87
|
+
# The gem's Zeitwerk loader, the app/models loader used outside Rails, and
|
|
88
|
+
# the configuration: built once when this file loads (see the bottom of
|
|
89
|
+
# the module body) and replaced only by reset!/teardown under @state_lock.
|
|
90
|
+
# Readers need no lock: a reference is published whole.
|
|
91
|
+
attr_reader :loader, :models_loader, :configuration
|
|
92
|
+
|
|
86
93
|
# Process-global flag set by Worker#graceful_shutdown so the adapter
|
|
87
94
|
# can report stopping? to ActiveJob::Continuation (Rails 8.1+).
|
|
88
|
-
attr_writer :stopping
|
|
89
|
-
|
|
90
95
|
def stopping
|
|
91
|
-
@stopping
|
|
96
|
+
@state_lock.synchronize { @stopping }
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def stopping=(value)
|
|
100
|
+
@state_lock.synchronize { @stopping = value ? true : false }
|
|
92
101
|
end
|
|
93
102
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
103
|
+
private
|
|
104
|
+
|
|
105
|
+
# True while a Rails app boots (its initializers and after_initialize
|
|
106
|
+
# blocks run): the engine finalizes at the end of that.
|
|
107
|
+
def rails_booting?
|
|
108
|
+
return false unless defined?(::Rails::Application) && ::Rails.respond_to?(:application)
|
|
109
|
+
|
|
110
|
+
app = ::Rails.application
|
|
111
|
+
!app.nil? && !app.initialized?
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Zeitwerk::Loader.for_gem works out the gem root from its caller, so this
|
|
115
|
+
# must stay in lib/pgbus.rb.
|
|
116
|
+
def build_loader
|
|
117
|
+
Zeitwerk::Loader.for_gem.tap do |loader|
|
|
97
118
|
loader.inflector.inflect(
|
|
98
119
|
"pgbus" => "Pgbus",
|
|
99
120
|
"cli" => "CLI",
|
|
@@ -127,35 +148,32 @@ module Pgbus
|
|
|
127
148
|
# root and tries to autoload Puma::Plugin, which collides with the real
|
|
128
149
|
# Puma::Plugin class defined by the puma gem itself.
|
|
129
150
|
loader.ignore("#{__dir__}/puma")
|
|
130
|
-
loader
|
|
131
151
|
end
|
|
132
152
|
end
|
|
133
153
|
|
|
134
154
|
# Separate loader for app/models used only in non-Rails contexts (specs,
|
|
135
155
|
# standalone scripts). When the Engine boots, Rails' autoloader takes over
|
|
136
156
|
# app/models and this loader is torn down to avoid conflicts.
|
|
137
|
-
def
|
|
157
|
+
def build_models_loader
|
|
138
158
|
models_dir = File.expand_path("../app/models", __dir__)
|
|
139
159
|
return nil unless File.directory?(models_dir)
|
|
140
160
|
|
|
141
|
-
|
|
142
|
-
loader = Zeitwerk::Loader.new
|
|
161
|
+
Zeitwerk::Loader.new.tap do |loader|
|
|
143
162
|
loader.tag = "pgbus-models"
|
|
144
163
|
loader.push_dir(models_dir)
|
|
145
164
|
loader.setup
|
|
146
|
-
loader
|
|
147
165
|
end
|
|
148
166
|
end
|
|
149
167
|
|
|
150
|
-
|
|
151
|
-
return unless @models_loader
|
|
152
|
-
|
|
153
|
-
@models_loader.unregister
|
|
154
|
-
@models_loader = nil
|
|
155
|
-
end
|
|
168
|
+
public
|
|
156
169
|
|
|
157
|
-
def
|
|
158
|
-
|
|
170
|
+
def teardown_models_loader!
|
|
171
|
+
loader = @state_lock.synchronize do
|
|
172
|
+
current = @models_loader
|
|
173
|
+
@models_loader = nil
|
|
174
|
+
current
|
|
175
|
+
end
|
|
176
|
+
loader&.unregister
|
|
159
177
|
end
|
|
160
178
|
|
|
161
179
|
# True when the current process is running a rake task (db:*, assets:*, …
|
|
@@ -188,11 +206,32 @@ module Pgbus
|
|
|
188
206
|
# until a worker path consumes it. Checking the flag after the yield
|
|
189
207
|
# lets a block opt out for itself via `c.eager_validation = false`.
|
|
190
208
|
configuration.validate! if configuration.eager_validation
|
|
209
|
+
# Without Rails this is the end of boot, so it finalizes; so does any
|
|
210
|
+
# configure once a Rails app has booted. While a Rails app boots, the
|
|
211
|
+
# engine's after_initialize finalizes instead, after every initializer
|
|
212
|
+
# and after_initialize block has had its say.
|
|
213
|
+
configuration.finalize! if configuration.finalized? || !rails_booting?
|
|
191
214
|
configuration
|
|
192
215
|
end
|
|
193
216
|
|
|
217
|
+
# The end of boot: validates the configuration (unless eager_validation
|
|
218
|
+
# is off) and freezes its plain-data settings into Ractor-shareable values
|
|
219
|
+
# (Configuration#finalize!). Callables and the logger stay as they are, on
|
|
220
|
+
# the main Ractor. Runs on its own: from the engine's after_initialize in a
|
|
221
|
+
# Rails app, at the end of each Pgbus.configure without Rails, and in
|
|
222
|
+
# `pgbus start` after its flags. Call it yourself only when you configure
|
|
223
|
+
# pgbus through setters alone, without Rails.
|
|
224
|
+
def finalize!
|
|
225
|
+
configuration.validate! if configuration.eager_validation
|
|
226
|
+
configuration.finalize!
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Built on first use, under @state_lock: two threads making the first
|
|
230
|
+
# call at once must not each build a Client (the loser's pool would be
|
|
231
|
+
# orphaned, its connections never closed). Not built at boot: a Client
|
|
232
|
+
# resolves connection options, which a rake db: task must not need.
|
|
194
233
|
def client
|
|
195
|
-
@client ||= Client.new(configuration)
|
|
234
|
+
@state_lock.synchronize { @client ||= Client.new(configuration) }
|
|
196
235
|
end
|
|
197
236
|
|
|
198
237
|
# Entry point for the streams subsystem — `Pgbus.stream(name).broadcast(html)`
|
|
@@ -218,8 +257,7 @@ module Pgbus
|
|
|
218
257
|
name = Streams::Stream.name_from(streamables)
|
|
219
258
|
resolved = durable.nil? ? configuration.stream_durable?(name) : durable
|
|
220
259
|
cache_key = "#{name}:#{resolved ? "d" : "e"}"
|
|
221
|
-
|
|
222
|
-
@stream_cache.compute_if_absent(cache_key) { Streams::Stream.new(streamables, durable: resolved) }
|
|
260
|
+
stream_cache.compute_if_absent(cache_key) { Streams::Stream.new(streamables, durable: resolved) }
|
|
223
261
|
end
|
|
224
262
|
|
|
225
263
|
# Compose a short, pgbus-safe stream identifier from any mix of
|
|
@@ -278,11 +316,14 @@ module Pgbus
|
|
|
278
316
|
EventBus::Publisher.publish_later(routing_key, payload, delay: delay, headers: headers)
|
|
279
317
|
end
|
|
280
318
|
|
|
319
|
+
# A fresh configuration (never nil, so readers need no lock) and no client.
|
|
281
320
|
def reset!
|
|
282
|
-
@
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
321
|
+
@state_lock.synchronize do
|
|
322
|
+
@client&.close
|
|
323
|
+
@client = nil
|
|
324
|
+
@configuration = Configuration.new
|
|
325
|
+
@stream_cache = Concurrent::Map.new
|
|
326
|
+
end
|
|
286
327
|
end
|
|
287
328
|
|
|
288
329
|
# Discard the inherited PGMQ client after fork.
|
|
@@ -290,9 +331,13 @@ module Pgbus
|
|
|
290
331
|
# state post-fork and attempting to acquire it can deadlock.
|
|
291
332
|
# The next call to Pgbus.client will lazily create a fresh one.
|
|
292
333
|
# Cached streams hold the client they were built with, so they go too.
|
|
334
|
+
# (@state_lock itself is safe to take in the child: Ruby abandons a Mutex
|
|
335
|
+
# whose owning thread did not survive the fork.)
|
|
293
336
|
def reset_client!
|
|
294
|
-
@
|
|
295
|
-
|
|
337
|
+
@state_lock.synchronize do
|
|
338
|
+
@client = nil
|
|
339
|
+
@stream_cache = Concurrent::Map.new
|
|
340
|
+
end
|
|
296
341
|
end
|
|
297
342
|
|
|
298
343
|
def logger
|
|
@@ -302,13 +347,30 @@ module Pgbus
|
|
|
302
347
|
def logger=(value)
|
|
303
348
|
configuration.logger = value
|
|
304
349
|
end
|
|
350
|
+
|
|
351
|
+
private
|
|
352
|
+
|
|
353
|
+
# Process-local, like the client it holds; replaced (never mutated in
|
|
354
|
+
# place) by reset!/reset_client!.
|
|
355
|
+
attr_reader :stream_cache
|
|
305
356
|
end
|
|
306
357
|
|
|
307
|
-
|
|
358
|
+
# Module state, built here at load rather than by `||=` on first use, so no
|
|
359
|
+
# two threads can each build their own (issue #527). @state_lock guards every
|
|
360
|
+
# later write: the client memo, reset!/reset_client!, the stopping flag.
|
|
361
|
+
@state_lock = Mutex.new
|
|
362
|
+
@stopping = false
|
|
363
|
+
@client = nil
|
|
364
|
+
@stream_cache = Concurrent::Map.new
|
|
365
|
+
@loader = build_loader
|
|
366
|
+
@loader.setup
|
|
367
|
+
|
|
368
|
+
# After loader.setup: Configuration is Zeitwerk-managed.
|
|
369
|
+
@configuration = Configuration.new
|
|
308
370
|
|
|
309
371
|
# In non-Rails contexts, set up model autoloading via a separate loader.
|
|
310
372
|
# This is torn down by the Engine initializer when Rails boots.
|
|
311
|
-
models_loader
|
|
373
|
+
@models_loader = defined?(Rails::Engine) ? nil : build_models_loader
|
|
312
374
|
end
|
|
313
375
|
|
|
314
376
|
require "active_job/queue_adapters/pgbus_adapter" if defined?(ActiveJob)
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
module Pgbus
|
|
2
|
+
module ActiveJob
|
|
3
|
+
# What the queue adapter delegates to.
|
|
4
|
+
class Adapter
|
|
5
|
+
def enqueue: (untyped active_job) -> untyped
|
|
6
|
+
def enqueue_at: (untyped active_job, Numeric timestamp) -> untyped
|
|
7
|
+
def enqueue_all: (Array[untyped] active_jobs) -> Integer
|
|
8
|
+
end
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
# The job-class options, included on ActiveJob::Base by the engine.
|
|
12
|
+
module Concurrency
|
|
13
|
+
module ClassMethods
|
|
14
|
+
def limits_concurrency: (to: Integer, ?key: untyped, ?duration: Numeric, ?on_conflict: :block | :discard | :raise) -> untyped
|
|
15
|
+
def pgbus_concurrency: () -> Hash[Symbol, untyped]?
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
module Uniqueness
|
|
20
|
+
module ClassMethods
|
|
21
|
+
# Any other keyword raises ArgumentError (lock_ttl: with a pointer to 1.0's removal).
|
|
22
|
+
def ensures_uniqueness: (?strategy: :until_executed | :while_executing, ?key: untyped,
|
|
23
|
+
?on_conflict: :reject | :discard | :log) -> untyped
|
|
24
|
+
def pgbus_uniqueness: () -> Hash[Symbol, untyped]?
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
module RetryBackoff
|
|
29
|
+
module JobMixin
|
|
30
|
+
module ClassMethods
|
|
31
|
+
def pgbus_retry_backoff: (?base: Numeric?, ?max: Numeric?, ?jitter: Numeric?) -> untyped
|
|
32
|
+
def pgbus_retry_backoff_config: () -> Hash[Symbol, Numeric]?
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
module VisibilityHeartbeat
|
|
38
|
+
module JobMixin
|
|
39
|
+
module ClassMethods
|
|
40
|
+
def pgbus_visibility_heartbeat: (?boolish enabled) -> bool
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
module ActiveJob
|
|
47
|
+
module QueueAdapters
|
|
48
|
+
# `config.active_job.queue_adapter = :pgbus`
|
|
49
|
+
class PgbusAdapter
|
|
50
|
+
def enqueue: (untyped active_job) -> untyped
|
|
51
|
+
def enqueue_at: (untyped active_job, Numeric timestamp) -> untyped
|
|
52
|
+
def enqueue_all: (Array[untyped] active_jobs) -> Integer
|
|
53
|
+
def enqueue_after_transaction_commit?: () -> bool
|
|
54
|
+
def stopping?: () -> bool
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|