pgbus 0.9.6 → 0.9.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +38 -0
- data/README.md +119 -1
- data/Rakefile +10 -1
- data/app/helpers/pgbus/application_helper.rb +45 -6
- data/app/views/pgbus/jobs/show.html.erb +1 -1
- data/app/views/pgbus/processes/_processes_table.html.erb +4 -1
- data/config/locales/da.yml +4 -0
- data/config/locales/de.yml +4 -0
- data/config/locales/en.yml +4 -0
- data/config/locales/es.yml +4 -0
- data/config/locales/fi.yml +4 -0
- data/config/locales/fr.yml +4 -0
- data/config/locales/it.yml +4 -0
- data/config/locales/ja.yml +4 -0
- data/config/locales/nb.yml +4 -0
- data/config/locales/nl.yml +4 -0
- data/config/locales/pt.yml +4 -0
- data/config/locales/sv.yml +4 -0
- data/lib/pgbus/active_job/executor.rb +25 -4
- data/lib/pgbus/cli/dlq.rb +164 -0
- data/lib/pgbus/cli.rb +18 -1
- data/lib/pgbus/client/connection_health.rb +194 -0
- data/lib/pgbus/client.rb +592 -73
- data/lib/pgbus/config_loader.rb +23 -4
- data/lib/pgbus/configuration.rb +102 -13
- data/lib/pgbus/dedup_cache.rb +8 -0
- data/lib/pgbus/doctor.rb +250 -0
- data/lib/pgbus/engine.rb +15 -0
- data/lib/pgbus/execution_pools/async_pool.rb +7 -0
- data/lib/pgbus/execution_pools/thread_pool.rb +7 -0
- data/lib/pgbus/instrumentation.rb +1 -0
- data/lib/pgbus/integrations/appsignal/probe.rb +23 -1
- data/lib/pgbus/metrics/backend.rb +38 -0
- data/lib/pgbus/metrics/backends/prometheus.rb +123 -0
- data/lib/pgbus/metrics/backends/statsd.rb +64 -0
- data/lib/pgbus/metrics/prometheus_exporter.rb +34 -0
- data/lib/pgbus/metrics/subscriber.rb +190 -0
- data/lib/pgbus/metrics.rb +42 -0
- data/lib/pgbus/process/consumer.rb +215 -8
- data/lib/pgbus/process/consumer_priority.rb +34 -0
- data/lib/pgbus/process/dispatcher.rb +265 -41
- data/lib/pgbus/process/heartbeat.rb +18 -5
- data/lib/pgbus/process/memory_usage.rb +48 -0
- data/lib/pgbus/process/notify_listener.rb +26 -7
- data/lib/pgbus/process/notify_probe.rb +96 -0
- data/lib/pgbus/process/primary_validator.rb +53 -0
- data/lib/pgbus/process/signal_handler.rb +6 -0
- data/lib/pgbus/process/supervisor.rb +396 -46
- data/lib/pgbus/process/worker.rb +298 -35
- data/lib/pgbus/recurring/scheduler.rb +15 -1
- data/lib/pgbus/streams/turbo_broadcastable.rb +7 -5
- data/lib/pgbus/table_maintenance.rb +13 -2
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source.rb +20 -4
- data/lib/pgbus/web/health_app.rb +102 -0
- data/lib/pgbus/web/health_server.rb +144 -0
- data/lib/pgbus/web/payload_filter.rb +68 -0
- data/lib/pgbus/web/streamer/instance.rb +55 -1
- data/lib/pgbus/web/streamer/listener.rb +72 -9
- data/lib/pgbus.rb +37 -0
- data/lib/rubocop/cop/pgbus/no_ruby_timeout.rb +42 -0
- data/lib/rubocop/pgbus.rb +5 -0
- data/lib/tasks/pgbus_doctor.rake +12 -0
- metadata +19 -1
data/lib/pgbus/config_loader.rb
CHANGED
|
@@ -11,16 +11,35 @@ module Pgbus
|
|
|
11
11
|
env ||= (defined?(Rails) && Rails.respond_to?(:env) && Rails.env) || ENV.fetch("PGBUS_ENV", "development")
|
|
12
12
|
raw = File.read(path)
|
|
13
13
|
parsed = YAML.safe_load(ERB.new(raw).result, permitted_classes: [Symbol], aliases: true)
|
|
14
|
-
|
|
15
|
-
|
|
14
|
+
# Distinguish sectioned (top-level env keys mapping to Hashes) from
|
|
15
|
+
# flat (top-level setter keys mapping to scalars/arrays). parsed.key?(env)
|
|
16
|
+
# alone can't tell them apart, so a flat file silently lost its typo
|
|
17
|
+
# warnings whenever no env section happened to match its keys.
|
|
18
|
+
if sectioned?(parsed)
|
|
19
|
+
apply(parsed.fetch(env)) if parsed.key?(env)
|
|
20
|
+
else
|
|
21
|
+
apply(parsed)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def sectioned?(parsed)
|
|
26
|
+
parsed.is_a?(Hash) && parsed.any? && parsed.values.all?(Hash)
|
|
16
27
|
end
|
|
17
28
|
|
|
18
|
-
def apply(hash)
|
|
29
|
+
def apply(hash, warn_unknown: true)
|
|
19
30
|
config = Pgbus.configuration
|
|
20
31
|
hash.each do |key, value|
|
|
21
32
|
setter = :"#{key}="
|
|
22
|
-
|
|
33
|
+
if config.respond_to?(setter)
|
|
34
|
+
config.public_send(setter, value)
|
|
35
|
+
elsif warn_unknown
|
|
36
|
+
config.logger.warn { "[Pgbus] Unknown configuration key ignored: #{key.inspect} — check for typos in pgbus.yml" }
|
|
37
|
+
end
|
|
23
38
|
end
|
|
39
|
+
# Validate eagerly so a bad YAML value aborts boot with an ArgumentError
|
|
40
|
+
# naming the offending key, instead of failing later in a worker path.
|
|
41
|
+
# Honors an `eager_validation: false` key in the same hash (applied above).
|
|
42
|
+
config.validate! if config.eager_validation
|
|
24
43
|
config
|
|
25
44
|
end
|
|
26
45
|
end
|
data/lib/pgbus/configuration.rb
CHANGED
|
@@ -86,7 +86,7 @@ module Pgbus
|
|
|
86
86
|
attr_reader :pgmq_schema_mode
|
|
87
87
|
|
|
88
88
|
# Event consumers
|
|
89
|
-
|
|
89
|
+
attr_reader :event_consumers
|
|
90
90
|
|
|
91
91
|
# Recurring jobs
|
|
92
92
|
attr_accessor :recurring_tasks, :recurring_schedule_interval, :recurring_tasks_file, :skip_recurring
|
|
@@ -103,13 +103,21 @@ module Pgbus
|
|
|
103
103
|
attr_accessor :zombie_detection
|
|
104
104
|
|
|
105
105
|
# Job stats
|
|
106
|
-
attr_accessor :stats_enabled
|
|
106
|
+
attr_accessor :stats_enabled, :stats_flush_size, :stats_flush_interval
|
|
107
107
|
attr_reader :stats_retention # rubocop:disable Style/AccessorGrouping
|
|
108
108
|
|
|
109
109
|
# Web dashboard
|
|
110
110
|
attr_accessor :web_auth, :web_refresh_interval, :web_per_page, :web_live_updates, :web_data_source,
|
|
111
111
|
:insights_default_minutes, :base_controller_class, :return_to_app_url,
|
|
112
|
-
:metrics_enabled
|
|
112
|
+
:metrics_enabled,
|
|
113
|
+
:dashboard_filter_parameters, :dashboard_filter_sensitive
|
|
114
|
+
|
|
115
|
+
# HTTP health endpoints (liveness/readiness for orchestrators like
|
|
116
|
+
# Kubernetes). `health_port` nil (default) leaves the standalone server in
|
|
117
|
+
# the supervisor disabled — mount Pgbus::Web::HealthApp in the host Rails
|
|
118
|
+
# app instead. When set, Supervisor#run serves /livez and /readyz over a
|
|
119
|
+
# plain TCPServer bound to `health_bind` (default localhost).
|
|
120
|
+
attr_accessor :health_port, :health_bind
|
|
113
121
|
|
|
114
122
|
# Streams (turbo-rails replacement, SSE-based)
|
|
115
123
|
attr_accessor :streams_enabled, :streams_path, :streams_queue_prefix, :streams_signed_name_secret,
|
|
@@ -135,6 +143,21 @@ module Pgbus
|
|
|
135
143
|
# Set to false to opt out without uninstalling the appsignal gem.
|
|
136
144
|
attr_accessor :appsignal_enabled, :appsignal_probe_enabled
|
|
137
145
|
|
|
146
|
+
# Generic metrics adapter (Pgbus::Metrics). Consumes the same pgbus.*
|
|
147
|
+
# instrumentation events as AppSignal and forwards them to a backend:
|
|
148
|
+
# nil (default) — install nothing, zero overhead
|
|
149
|
+
# :prometheus — in-process registry, scraped via PrometheusExporter
|
|
150
|
+
# :statsd — UDP datagrams to statsd_host:statsd_port
|
|
151
|
+
# a Pgbus::Metrics::Backend instance — a custom backend
|
|
152
|
+
attr_accessor :metrics_backend, :statsd_host, :statsd_port
|
|
153
|
+
|
|
154
|
+
# When true (default), Pgbus.configure and ConfigLoader.apply run
|
|
155
|
+
# validate! after applying settings, so an invalid value fails loud at
|
|
156
|
+
# boot instead of dormant until a worker path consumes it. Set false to
|
|
157
|
+
# opt out (exotic setups that intentionally hold a transiently-invalid
|
|
158
|
+
# config); explicit validate! still works.
|
|
159
|
+
attr_accessor :eager_validation
|
|
160
|
+
|
|
138
161
|
def initialize
|
|
139
162
|
@database_url = nil
|
|
140
163
|
@connection_params = nil
|
|
@@ -208,6 +231,11 @@ module Pgbus
|
|
|
208
231
|
|
|
209
232
|
@stats_enabled = true
|
|
210
233
|
@stats_retention = 30 * 24 * 3600 # 30 days
|
|
234
|
+
# StatBuffer flush thresholds. Buffered stats are lost on SIGKILL, so a
|
|
235
|
+
# smaller size/interval shrinks the residual loss window at the cost of
|
|
236
|
+
# more frequent bulk inserts. Defaults mirror StatBuffer's own constants.
|
|
237
|
+
@stats_flush_size = StatBuffer::DEFAULT_FLUSH_SIZE
|
|
238
|
+
@stats_flush_interval = StatBuffer::DEFAULT_FLUSH_INTERVAL
|
|
211
239
|
|
|
212
240
|
@connects_to = nil
|
|
213
241
|
|
|
@@ -220,6 +248,12 @@ module Pgbus
|
|
|
220
248
|
@base_controller_class = "::ActionController::Base"
|
|
221
249
|
@return_to_app_url = nil
|
|
222
250
|
@metrics_enabled = true
|
|
251
|
+
@dashboard_filter_parameters = nil # nil = auto-detect from Rails, then fall back to defaults
|
|
252
|
+
@dashboard_filter_sensitive = true
|
|
253
|
+
|
|
254
|
+
# HTTP health endpoints — nil port disables the standalone server.
|
|
255
|
+
@health_port = nil
|
|
256
|
+
@health_bind = "127.0.0.1"
|
|
223
257
|
|
|
224
258
|
@streams_enabled = true
|
|
225
259
|
@streams_path = nil
|
|
@@ -284,6 +318,13 @@ module Pgbus
|
|
|
284
318
|
# the same process, so the operator can disable it independently.
|
|
285
319
|
@appsignal_enabled = true
|
|
286
320
|
@appsignal_probe_enabled = true
|
|
321
|
+
|
|
322
|
+
# Generic metrics adapter: off by default (no subscription installed).
|
|
323
|
+
@metrics_backend = nil
|
|
324
|
+
@statsd_host = "127.0.0.1"
|
|
325
|
+
@statsd_port = 8125
|
|
326
|
+
|
|
327
|
+
@eager_validation = true
|
|
287
328
|
end
|
|
288
329
|
|
|
289
330
|
def queue_name(name)
|
|
@@ -308,7 +349,7 @@ module Pgbus
|
|
|
308
349
|
# Returns the execution mode for a specific worker config hash,
|
|
309
350
|
# falling back to the global execution_mode setting.
|
|
310
351
|
def execution_mode_for(worker_config)
|
|
311
|
-
mode = worker_config[:execution_mode] ||
|
|
352
|
+
mode = worker_config[:execution_mode] || execution_mode
|
|
312
353
|
ExecutionPools.normalize_mode(mode)
|
|
313
354
|
end
|
|
314
355
|
|
|
@@ -390,15 +431,26 @@ module Pgbus
|
|
|
390
431
|
raise ArgumentError, "read_timeout must be a positive number or nil to disable"
|
|
391
432
|
end
|
|
392
433
|
|
|
434
|
+
unless stats_flush_size.is_a?(Integer) && stats_flush_size.positive?
|
|
435
|
+
raise ArgumentError, "stats_flush_size must be a positive integer"
|
|
436
|
+
end
|
|
437
|
+
unless stats_flush_interval.is_a?(Numeric) && stats_flush_interval.positive?
|
|
438
|
+
raise ArgumentError, "stats_flush_interval must be a positive number"
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
unless health_port.nil? || (health_port.is_a?(Integer) && health_port.between?(1, 65_535))
|
|
442
|
+
raise ArgumentError, "health_port must be an integer between 1 and 65535 or nil to disable"
|
|
443
|
+
end
|
|
444
|
+
|
|
393
445
|
# Validate global execution_mode
|
|
394
446
|
ExecutionPools.normalize_mode(execution_mode)
|
|
395
447
|
|
|
396
448
|
Array(workers).each do |w|
|
|
397
|
-
threads = w[:threads] ||
|
|
449
|
+
threads = w[:threads] || 5
|
|
398
450
|
raise ArgumentError, "worker threads must be > 0" unless threads.is_a?(Integer) && threads.positive?
|
|
399
451
|
|
|
400
452
|
# Validate per-worker execution_mode override if present
|
|
401
|
-
mode = w[:execution_mode]
|
|
453
|
+
mode = w[:execution_mode]
|
|
402
454
|
ExecutionPools.normalize_mode(mode) if mode
|
|
403
455
|
end
|
|
404
456
|
|
|
@@ -413,6 +465,7 @@ module Pgbus
|
|
|
413
465
|
end
|
|
414
466
|
|
|
415
467
|
validate_streams!
|
|
468
|
+
validate_metrics_backend!
|
|
416
469
|
|
|
417
470
|
self
|
|
418
471
|
end
|
|
@@ -462,6 +515,17 @@ module Pgbus
|
|
|
462
515
|
raise ArgumentError, "streams_orphan_threshold must be a positive number or nil to disable"
|
|
463
516
|
end
|
|
464
517
|
|
|
518
|
+
def validate_metrics_backend!
|
|
519
|
+
value = metrics_backend
|
|
520
|
+
return if value.nil?
|
|
521
|
+
return if %i[prometheus statsd].include?(value)
|
|
522
|
+
return if defined?(Pgbus::Metrics::Backend) && value.is_a?(Pgbus::Metrics::Backend)
|
|
523
|
+
|
|
524
|
+
raise ArgumentError,
|
|
525
|
+
"metrics_backend must be nil, :prometheus, :statsd, or a " \
|
|
526
|
+
"Pgbus::Metrics::Backend instance (got #{value.inspect})"
|
|
527
|
+
end
|
|
528
|
+
|
|
465
529
|
# Returns true if the given stream name should be durable based on
|
|
466
530
|
# `streams_durable_patterns` (exact string or Regexp match) or the
|
|
467
531
|
# `streams_default_broadcast_mode` fallback.
|
|
@@ -549,13 +613,29 @@ module Pgbus
|
|
|
549
613
|
parsed = CapsuleDSL.parse(value)
|
|
550
614
|
assign_auto_names(parsed)
|
|
551
615
|
when Array
|
|
552
|
-
value
|
|
616
|
+
value.map { |entry| normalize_entry(entry, group: "worker") }
|
|
553
617
|
else
|
|
554
618
|
raise ArgumentError,
|
|
555
619
|
"workers must be a String (DSL), Array (legacy form), or nil — got #{value.class}"
|
|
556
620
|
end
|
|
557
621
|
end
|
|
558
622
|
|
|
623
|
+
# Event consumer configs arrive with symbol keys (Ruby DSL) or string keys
|
|
624
|
+
# (YAML via ConfigLoader). Normalize once here so every read site uses
|
|
625
|
+
# symbol access only. nil passes through (no consumers configured).
|
|
626
|
+
def event_consumers=(value)
|
|
627
|
+
@event_consumers =
|
|
628
|
+
case value
|
|
629
|
+
when nil
|
|
630
|
+
nil
|
|
631
|
+
when Array
|
|
632
|
+
value.map { |entry| normalize_entry(entry, group: "event_consumer") }
|
|
633
|
+
else
|
|
634
|
+
raise ArgumentError,
|
|
635
|
+
"event_consumers must be an Array or nil — got #{value.class}"
|
|
636
|
+
end
|
|
637
|
+
end
|
|
638
|
+
|
|
559
639
|
# Define a named capsule and append it to the workers list.
|
|
560
640
|
#
|
|
561
641
|
# c.capsule :critical, queues: %w[critical], threads: 5
|
|
@@ -847,11 +927,20 @@ module Pgbus
|
|
|
847
927
|
numeric
|
|
848
928
|
end
|
|
849
929
|
|
|
850
|
-
#
|
|
851
|
-
#
|
|
930
|
+
# Symbolize the top-level keys of a worker/consumer config entry so every
|
|
931
|
+
# downstream read site can use symbol access only. YAML (via ConfigLoader)
|
|
932
|
+
# yields string keys; the Ruby DSL and +capsule+ builder yield symbols.
|
|
933
|
+
# Entries are flat hashes with array/scalar values — no deep recursion.
|
|
934
|
+
def normalize_entry(entry, group:)
|
|
935
|
+
raise ArgumentError, "#{group} entry must be a Hash, got #{entry.class}: #{entry.inspect}" unless entry.is_a?(Hash)
|
|
936
|
+
|
|
937
|
+
entry.transform_keys(&:to_sym)
|
|
938
|
+
end
|
|
939
|
+
|
|
940
|
+
# Read a capsule's name (symbol key after normalization), as a string for
|
|
941
|
+
# comparison. Returns nil for unnamed (legacy) entries.
|
|
852
942
|
def capsule_name(entry)
|
|
853
|
-
|
|
854
|
-
raw&.to_s
|
|
943
|
+
entry[:name]&.to_s
|
|
855
944
|
end
|
|
856
945
|
|
|
857
946
|
# Auto-assign :name to parsed capsules where the first queue token would
|
|
@@ -884,7 +973,7 @@ module Pgbus
|
|
|
884
973
|
existing_named = (@workers || []).select { |c| capsule_name(c) }
|
|
885
974
|
return if existing_named.empty?
|
|
886
975
|
|
|
887
|
-
existing_queues = existing_named.flat_map { |c| c[:queues] ||
|
|
976
|
+
existing_queues = existing_named.flat_map { |c| c[:queues] || [] }
|
|
888
977
|
return if existing_queues.empty?
|
|
889
978
|
|
|
890
979
|
if existing_queues.include?(CapsuleDSL::WILDCARD)
|
|
@@ -912,7 +1001,7 @@ module Pgbus
|
|
|
912
1001
|
return 0 unless entries
|
|
913
1002
|
|
|
914
1003
|
entries.sum do |entry|
|
|
915
|
-
threads = entry[:threads] ||
|
|
1004
|
+
threads = entry[:threads] || default_threads
|
|
916
1005
|
unless threads.is_a?(Integer) && threads.positive?
|
|
917
1006
|
raise ArgumentError,
|
|
918
1007
|
"#{group} threads must be a positive integer, got #{threads.inspect}"
|
data/lib/pgbus/dedup_cache.rb
CHANGED
|
@@ -60,11 +60,19 @@ module Pgbus
|
|
|
60
60
|
|
|
61
61
|
def evict(key)
|
|
62
62
|
@cache.delete(key)
|
|
63
|
+
# Concurrent::Array#delete is O(n), but this only runs on expiry and n is
|
|
64
|
+
# bounded by @max_size, so it stays cheap. Without it, @insertion_order
|
|
65
|
+
# leaks and stale duplicates can evict live keys in evict_oldest.
|
|
66
|
+
@insertion_order.delete(key)
|
|
63
67
|
end
|
|
64
68
|
|
|
65
69
|
def evict_oldest
|
|
66
70
|
while @cache.size >= @max_size && !@insertion_order.empty?
|
|
67
71
|
oldest_key = @insertion_order.shift
|
|
72
|
+
# Skip residual order entries for keys already gone from @cache so a
|
|
73
|
+
# stale duplicate never evicts a freshly re-marked (live) key.
|
|
74
|
+
next unless @cache.key?(oldest_key)
|
|
75
|
+
|
|
68
76
|
@cache.delete(oldest_key)
|
|
69
77
|
end
|
|
70
78
|
end
|
data/lib/pgbus/doctor.rb
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "uri"
|
|
4
|
+
require "pgbus/mcp/health_analyzer"
|
|
5
|
+
|
|
6
|
+
module Pgbus
|
|
7
|
+
# Preflight diagnostics for a pgbus deployment — the single command that
|
|
8
|
+
# answers "is this environment healthy enough to run?". Runs six checks and
|
|
9
|
+
# returns a machine-readable result plus a human report, so `pgbus doctor`
|
|
10
|
+
# and `rake pgbus:doctor` can gate a deploy or CI run (exit 0 on success,
|
|
11
|
+
# 1 on any failure).
|
|
12
|
+
#
|
|
13
|
+
# Doctor never touches PGMQ or PostgreSQL directly: every probe goes through
|
|
14
|
+
# Pgbus::Client (DB, PGMQ schema, queues, NOTIFY) or Pgbus::Web::DataSource
|
|
15
|
+
# (process liveness, via Pgbus::MCP::HealthAnalyzer). It also never raises —
|
|
16
|
+
# a broken environment turns into :fail results, never a crash — so it is
|
|
17
|
+
# safe to run against a database that is down or a half-installed schema.
|
|
18
|
+
class Doctor
|
|
19
|
+
# A single check result. status is one of :ok, :warn, :fail.
|
|
20
|
+
Check = Struct.new(:name, :status, :detail) do
|
|
21
|
+
def to_h
|
|
22
|
+
{ name: name, status: status, detail: detail }
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
STATUS_ICON = { ok: "✓", warn: "!", fail: "✗" }.freeze
|
|
27
|
+
|
|
28
|
+
def initialize(config: Pgbus.configuration, client: Pgbus.client, data_source: nil)
|
|
29
|
+
@config = config
|
|
30
|
+
@client = client
|
|
31
|
+
@data_source = data_source || Pgbus::Web::DataSource.new(client: client)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Run all checks and return an array of result hashes:
|
|
35
|
+
# { name:, status: :ok|:warn|:fail, detail: }
|
|
36
|
+
def run
|
|
37
|
+
@run ||= [
|
|
38
|
+
check_configuration,
|
|
39
|
+
check_database,
|
|
40
|
+
check_pgmq_schema,
|
|
41
|
+
check_queues,
|
|
42
|
+
check_notify,
|
|
43
|
+
check_processes
|
|
44
|
+
].map(&:to_h)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# True when no check failed. Warnings do not fail the run — they surface a
|
|
48
|
+
# concern (e.g. an outdated PGMQ schema) without blocking a deploy.
|
|
49
|
+
def success?
|
|
50
|
+
run.none? { |c| c[:status] == :fail }
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Human-readable report: one line per check, then a resolved-config summary
|
|
54
|
+
# with passwords redacted. Suitable for stdout in the CLI and rake task.
|
|
55
|
+
def report
|
|
56
|
+
lines = ["Pgbus Doctor", "=" * 40]
|
|
57
|
+
run.each do |check|
|
|
58
|
+
icon = STATUS_ICON.fetch(check[:status], "?")
|
|
59
|
+
lines << format("%<icon>s %<name>-22s %<detail>s", icon: icon, name: check[:name], detail: check[:detail])
|
|
60
|
+
end
|
|
61
|
+
lines << ""
|
|
62
|
+
lines << "Configuration"
|
|
63
|
+
lines << ("-" * 40)
|
|
64
|
+
config_summary.each { |key, value| lines << format(" %<key>-20s %<value>s", key: key, value: value) }
|
|
65
|
+
lines.join("\n")
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Resolved-config summary for the report. Password-bearing fields
|
|
69
|
+
# (database_url, connection_params) are redacted.
|
|
70
|
+
def config_summary
|
|
71
|
+
{
|
|
72
|
+
queue_prefix: @config.queue_prefix,
|
|
73
|
+
default_queue: @config.default_queue,
|
|
74
|
+
pgmq_schema_mode: @config.pgmq_schema_mode,
|
|
75
|
+
resolved_pool_size: safe { @config.resolved_pool_size },
|
|
76
|
+
listen_notify: @config.listen_notify,
|
|
77
|
+
roles: @config.roles || "all",
|
|
78
|
+
capsules: capsule_summary,
|
|
79
|
+
database_url: redacted_database_url,
|
|
80
|
+
connection_params: redacted_connection_params
|
|
81
|
+
}.compact
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
private
|
|
85
|
+
|
|
86
|
+
# 1. Configuration validity.
|
|
87
|
+
def check_configuration
|
|
88
|
+
@config.validate!
|
|
89
|
+
Check.new(name: "Configuration", status: :ok, detail: "valid")
|
|
90
|
+
rescue ArgumentError => e
|
|
91
|
+
Check.new(name: "Configuration", status: :fail, detail: e.message)
|
|
92
|
+
rescue StandardError => e
|
|
93
|
+
Check.new(name: "Configuration", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# 2. Database connectivity — SELECT 1 via Client#ping.
|
|
97
|
+
def check_database
|
|
98
|
+
@client.ping
|
|
99
|
+
Check.new(name: "Database", status: :ok, detail: "reachable")
|
|
100
|
+
rescue StandardError => e
|
|
101
|
+
Check.new(name: "Database", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# 3. PGMQ schema presence + installed-vs-vendored version.
|
|
105
|
+
#
|
|
106
|
+
# A nil version does NOT automatically mean "not installed": PGMQ installed
|
|
107
|
+
# via the extension, or before pgbus added version tracking, leaves the
|
|
108
|
+
# pgmq schema fully working with no row in pgbus_pgmq_schema_versions. So we
|
|
109
|
+
# distinguish, mirroring `pgbus:pgmq:status`: schema absent → fail; schema
|
|
110
|
+
# present but untracked → warn; tracked but behind → warn; current → ok.
|
|
111
|
+
def check_pgmq_schema
|
|
112
|
+
installed = @client.pgmq_schema_version
|
|
113
|
+
latest = Pgbus::PgmqSchema.latest_version
|
|
114
|
+
|
|
115
|
+
if installed.nil?
|
|
116
|
+
unless @client.pgmq_installed?
|
|
117
|
+
return Check.new(name: "PGMQ schema", status: :fail,
|
|
118
|
+
detail: "not installed — run `rails generate pgbus:install`")
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
return Check.new(name: "PGMQ schema", status: :warn,
|
|
122
|
+
detail: "installed but no version tracking — run `rails generate pgbus:upgrade_pgmq`")
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
if Gem::Version.new(installed) < Gem::Version.new(latest)
|
|
126
|
+
Check.new(name: "PGMQ schema", status: :warn,
|
|
127
|
+
detail: "installed #{installed}, vendored #{latest} — run `rails generate pgbus:upgrade_pgmq`")
|
|
128
|
+
else
|
|
129
|
+
Check.new(name: "PGMQ schema", status: :ok, detail: "up to date (#{installed})")
|
|
130
|
+
end
|
|
131
|
+
rescue StandardError => e
|
|
132
|
+
Check.new(name: "PGMQ schema", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# 4. Queue existence — every configured queue must have its PGMQ table(s).
|
|
136
|
+
# Resolve each logical queue to the PHYSICAL names bootstrap creates (via the
|
|
137
|
+
# client's queue strategy) so a priority queue's _p0.._pN sub-tables are what
|
|
138
|
+
# we diff against list_queues — not the bare prefixed name priority mode
|
|
139
|
+
# never creates.
|
|
140
|
+
def check_queues
|
|
141
|
+
configured = @client.configured_queues.flat_map { |q| @client.physical_queue_names(q) }
|
|
142
|
+
existing = existing_queue_names
|
|
143
|
+
missing = configured - existing
|
|
144
|
+
|
|
145
|
+
if missing.empty?
|
|
146
|
+
Check.new(name: "Queues", status: :ok, detail: "#{configured.size} configured queue(s) present")
|
|
147
|
+
else
|
|
148
|
+
Check.new(name: "Queues", status: :fail, detail: "missing queue(s): #{missing.join(", ")}")
|
|
149
|
+
end
|
|
150
|
+
rescue StandardError => e
|
|
151
|
+
Check.new(name: "Queues", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# 5. LISTEN/NOTIFY — when configured on, every configured queue should carry
|
|
155
|
+
# the insert-NOTIFY trigger. A missing trigger is a warning, not a failure:
|
|
156
|
+
# pgbus still works (it falls back to polling), it just loses instant wakeup.
|
|
157
|
+
def check_notify
|
|
158
|
+
return Check.new(name: "LISTEN/NOTIFY", status: :ok, detail: "disabled in config") unless @config.listen_notify
|
|
159
|
+
|
|
160
|
+
without_trigger = @client.configured_queues.reject { |q| @client.notify_enabled?(q) }
|
|
161
|
+
if without_trigger.empty?
|
|
162
|
+
Check.new(name: "LISTEN/NOTIFY", status: :ok, detail: "triggers live on all configured queues")
|
|
163
|
+
else
|
|
164
|
+
Check.new(name: "LISTEN/NOTIFY", status: :warn,
|
|
165
|
+
detail: "no insert trigger on: #{without_trigger.join(", ")} (falling back to polling)")
|
|
166
|
+
end
|
|
167
|
+
rescue StandardError => e
|
|
168
|
+
Check.new(name: "LISTEN/NOTIFY", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# 6. Process liveness — the top-level health verdict (OK/DEGRADED/STALLED)
|
|
172
|
+
# from the shared HealthAnalyzer. STALLED (the silent-worker-wedge) is a
|
|
173
|
+
# failure; DEGRADED is a warning; OK passes.
|
|
174
|
+
def check_processes
|
|
175
|
+
verdict = Pgbus::MCP::HealthAnalyzer.new(@data_source).verdict
|
|
176
|
+
status = verdict[:status]
|
|
177
|
+
detail = Array(verdict[:reasons]).first || status
|
|
178
|
+
|
|
179
|
+
case status
|
|
180
|
+
when "STALLED" then Check.new(name: "Process liveness", status: :fail, detail: detail)
|
|
181
|
+
when "DEGRADED" then Check.new(name: "Process liveness", status: :warn, detail: detail)
|
|
182
|
+
else Check.new(name: "Process liveness", status: :ok, detail: "OK")
|
|
183
|
+
end
|
|
184
|
+
rescue StandardError => e
|
|
185
|
+
Check.new(name: "Process liveness", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# Physical queue names known to PGMQ. list_queues rows may be Hashes (with a
|
|
189
|
+
# :queue_name key) or value objects responding to #queue_name.
|
|
190
|
+
def existing_queue_names
|
|
191
|
+
Array(@client.list_queues).map do |row|
|
|
192
|
+
if row.respond_to?(:queue_name)
|
|
193
|
+
row.queue_name
|
|
194
|
+
elsif row.is_a?(Hash)
|
|
195
|
+
row[:queue_name] || row["queue_name"]
|
|
196
|
+
end
|
|
197
|
+
end.compact
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def capsule_summary
|
|
201
|
+
Array(@config.workers).map { |w| w[:name] || (w[:queues] || []).join("+") }.join(", ")
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# Redact the password from a libpq URL/URI while keeping host/db visible so
|
|
205
|
+
# the operator can still confirm which database they pointed at.
|
|
206
|
+
def redacted_database_url
|
|
207
|
+
return nil unless @config.database_url
|
|
208
|
+
|
|
209
|
+
redact_url(@config.database_url)
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
# connection_params is a free-form libpq keyword hash, so redact every key
|
|
213
|
+
# whose name looks secret (password, sslpassword, ...) — not just :password —
|
|
214
|
+
# rather than assuming a fixed shape. Preserves symbol/string key form.
|
|
215
|
+
SECRET_KEY_PATTERN = /pass|secret|token/i
|
|
216
|
+
private_constant :SECRET_KEY_PATTERN
|
|
217
|
+
|
|
218
|
+
def redacted_connection_params
|
|
219
|
+
params = @config.connection_params
|
|
220
|
+
return nil unless params.is_a?(Hash)
|
|
221
|
+
|
|
222
|
+
params.each_with_object({}) do |(key, value), out|
|
|
223
|
+
out[key] = key.to_s.match?(SECRET_KEY_PATTERN) ? "[REDACTED]" : value
|
|
224
|
+
end
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# Replace the password in a userinfo authority (scheme://user:pass@host) or a
|
|
228
|
+
# key=value conninfo string (password=secret). Falls back to a blanket
|
|
229
|
+
# redaction label if parsing fails, never leaking the original.
|
|
230
|
+
#
|
|
231
|
+
# The userinfo password can itself contain '@' and ':' (common in generated
|
|
232
|
+
# secrets), so match it greedily up to the LAST '@' before the authority's
|
|
233
|
+
# host — a lazy `[^@]+` would stop at the first '@' and leak the remainder.
|
|
234
|
+
def redact_url(url)
|
|
235
|
+
redacted = url.sub(%r{(://[^:/@]+:).+(@[^@/]*(?:[/?]|\z))}, '\1[REDACTED]\2')
|
|
236
|
+
redacted.gsub(/(\bpassword=)('[^']*'|[^\s'"]+)/i, '\1[REDACTED]')
|
|
237
|
+
rescue StandardError
|
|
238
|
+
"[REDACTED]"
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
# Wrap a value-producing block so a broken resolver (e.g. resolved_pool_size
|
|
242
|
+
# raising on a malformed worker config) degrades to nil instead of raising
|
|
243
|
+
# out of config_summary.
|
|
244
|
+
def safe
|
|
245
|
+
yield
|
|
246
|
+
rescue StandardError
|
|
247
|
+
nil
|
|
248
|
+
end
|
|
249
|
+
end
|
|
250
|
+
end
|
data/lib/pgbus/engine.rb
CHANGED
|
@@ -60,6 +60,7 @@ module Pgbus
|
|
|
60
60
|
load File.expand_path("../tasks/pgbus_pgmq.rake", __dir__)
|
|
61
61
|
load File.expand_path("../tasks/pgbus_streams.rake", __dir__)
|
|
62
62
|
load File.expand_path("../tasks/pgbus_autovacuum.rake", __dir__)
|
|
63
|
+
load File.expand_path("../tasks/pgbus_doctor.rake", __dir__)
|
|
63
64
|
end
|
|
64
65
|
|
|
65
66
|
initializer "pgbus.i18n" do
|
|
@@ -86,6 +87,20 @@ module Pgbus
|
|
|
86
87
|
end
|
|
87
88
|
end
|
|
88
89
|
|
|
90
|
+
# Generic metrics adapter. Opt-in via config.metrics_backend; nil (default)
|
|
91
|
+
# installs no subscription, so there is zero overhead when unused. Runs
|
|
92
|
+
# independently of AppSignal — both consume the same pgbus.* events and both
|
|
93
|
+
# can be active at once.
|
|
94
|
+
initializer "pgbus.metrics", after: :load_config_initializers do
|
|
95
|
+
ActiveSupport.on_load(:after_initialize) do
|
|
96
|
+
next if Pgbus.configuration.metrics_backend.nil?
|
|
97
|
+
|
|
98
|
+
backend = Pgbus::Metrics.resolve_backend(Pgbus.configuration.metrics_backend)
|
|
99
|
+
Pgbus::Metrics::Subscriber.install!(backend: backend)
|
|
100
|
+
Pgbus.logger.info { "[Pgbus] Metrics subscriber installed (#{backend.class})" }
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
89
104
|
# Install the watermark cache middleware ahead of the app's own
|
|
90
105
|
# middleware so the thread-local cache is cleared between every
|
|
91
106
|
# Rack request. Without this, repeated page renders served by the
|
|
@@ -46,6 +46,13 @@ module Pgbus
|
|
|
46
46
|
available_capacity.positive?
|
|
47
47
|
end
|
|
48
48
|
|
|
49
|
+
# True only when every slot is free — all in-flight work has finished.
|
|
50
|
+
# idle? answers "can this pool accept work?"; quiesced? answers
|
|
51
|
+
# "has this pool drained?". The worker drain loop needs the latter.
|
|
52
|
+
def quiesced?
|
|
53
|
+
available_capacity >= @capacity
|
|
54
|
+
end
|
|
55
|
+
|
|
49
56
|
def shutdown
|
|
50
57
|
@state_mutex.synchronize do
|
|
51
58
|
return false if @shutdown_flag
|
|
@@ -37,6 +37,13 @@ module Pgbus
|
|
|
37
37
|
available_capacity.positive?
|
|
38
38
|
end
|
|
39
39
|
|
|
40
|
+
# True only when every slot is free — all in-flight work has finished.
|
|
41
|
+
# idle? answers "can this pool accept work?"; quiesced? answers
|
|
42
|
+
# "has this pool drained?". The worker drain loop needs the latter.
|
|
43
|
+
def quiesced?
|
|
44
|
+
available_capacity >= @capacity
|
|
45
|
+
end
|
|
46
|
+
|
|
40
47
|
def shutdown
|
|
41
48
|
@pool.shutdown
|
|
42
49
|
end
|
|
@@ -11,6 +11,7 @@ module Pgbus
|
|
|
11
11
|
# pgbus.client.send_batch — batch enqueue
|
|
12
12
|
# pgbus.client.read_batch — batch dequeue
|
|
13
13
|
# pgbus.client.read_message — single message dequeue
|
|
14
|
+
# pgbus.client.pool — connection pool snapshot (size/available/pool_timeout); emitted per worker heartbeat
|
|
14
15
|
# pgbus.executor.execute — full job execution (deserialize + perform + archive)
|
|
15
16
|
# pgbus.job_completed — job archived successfully
|
|
16
17
|
# pgbus.job_failed — job raised; carries :exception_object
|
|
@@ -51,8 +51,9 @@ module Pgbus
|
|
|
51
51
|
|
|
52
52
|
# The actual probe object; AppSignal calls #call once per minute.
|
|
53
53
|
class Runner
|
|
54
|
-
def initialize(data_source: nil)
|
|
54
|
+
def initialize(data_source: nil, client: nil)
|
|
55
55
|
@data_source = data_source
|
|
56
|
+
@client = client
|
|
56
57
|
@hostname = Socket.gethostname
|
|
57
58
|
end
|
|
58
59
|
|
|
@@ -63,6 +64,7 @@ module Pgbus
|
|
|
63
64
|
track_processes
|
|
64
65
|
track_summary
|
|
65
66
|
track_streams
|
|
67
|
+
track_pool
|
|
66
68
|
end
|
|
67
69
|
|
|
68
70
|
private
|
|
@@ -72,6 +74,10 @@ module Pgbus
|
|
|
72
74
|
(::Pgbus::Web::DataSource.new if defined?(::Pgbus::Web::DataSource))
|
|
73
75
|
end
|
|
74
76
|
|
|
77
|
+
def client
|
|
78
|
+
@client ||= (::Pgbus.client if defined?(::Pgbus) && ::Pgbus.respond_to?(:client))
|
|
79
|
+
end
|
|
80
|
+
|
|
75
81
|
def track_queues
|
|
76
82
|
data_source.queues_with_metrics.each do |q|
|
|
77
83
|
tags = { queue: q[:name] }
|
|
@@ -110,6 +116,22 @@ module Pgbus
|
|
|
110
116
|
log_failure("summary metrics", e)
|
|
111
117
|
end
|
|
112
118
|
|
|
119
|
+
# The PGMQ connection pool is per-process (each host owns its own
|
|
120
|
+
# pool), so — unlike the cluster-wide queue/summary gauges — these
|
|
121
|
+
# are tagged with the hostname, same policy as active_processes.
|
|
122
|
+
# pool_stats already rescues to {} internally; the outer rescue here
|
|
123
|
+
# keeps a probe iteration alive if the client itself is unavailable.
|
|
124
|
+
def track_pool
|
|
125
|
+
stats = client&.pool_stats || {}
|
|
126
|
+
return if stats.empty?
|
|
127
|
+
|
|
128
|
+
tags = { hostname: @hostname }
|
|
129
|
+
gauge "pool_size", stats[:size], tags
|
|
130
|
+
gauge "pool_available", stats[:available], tags
|
|
131
|
+
rescue StandardError => e
|
|
132
|
+
log_failure("pool metrics", e)
|
|
133
|
+
end
|
|
134
|
+
|
|
113
135
|
def track_streams
|
|
114
136
|
return unless data_source.respond_to?(:stream_stats_available?) &&
|
|
115
137
|
data_source.stream_stats_available?
|