pgbus 0.9.7 → 0.9.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +38 -0
- data/README.md +119 -1
- data/Rakefile +10 -1
- data/app/helpers/pgbus/application_helper.rb +37 -0
- data/app/views/pgbus/processes/_processes_table.html.erb +4 -1
- data/config/locales/da.yml +4 -0
- data/config/locales/de.yml +4 -0
- data/config/locales/en.yml +4 -0
- data/config/locales/es.yml +4 -0
- data/config/locales/fi.yml +4 -0
- data/config/locales/fr.yml +4 -0
- data/config/locales/it.yml +4 -0
- data/config/locales/ja.yml +4 -0
- data/config/locales/nb.yml +4 -0
- data/config/locales/nl.yml +4 -0
- data/config/locales/pt.yml +4 -0
- data/config/locales/sv.yml +4 -0
- data/lib/pgbus/active_job/executor.rb +25 -4
- data/lib/pgbus/cli/dlq.rb +164 -0
- data/lib/pgbus/cli.rb +18 -1
- data/lib/pgbus/client/connection_health.rb +194 -0
- data/lib/pgbus/client.rb +592 -73
- data/lib/pgbus/config_loader.rb +23 -4
- data/lib/pgbus/configuration.rb +98 -12
- data/lib/pgbus/dedup_cache.rb +8 -0
- data/lib/pgbus/doctor.rb +250 -0
- data/lib/pgbus/engine.rb +15 -0
- data/lib/pgbus/execution_pools/async_pool.rb +7 -0
- data/lib/pgbus/execution_pools/thread_pool.rb +7 -0
- data/lib/pgbus/instrumentation.rb +1 -0
- data/lib/pgbus/integrations/appsignal/probe.rb +23 -1
- data/lib/pgbus/metrics/backend.rb +38 -0
- data/lib/pgbus/metrics/backends/prometheus.rb +123 -0
- data/lib/pgbus/metrics/backends/statsd.rb +64 -0
- data/lib/pgbus/metrics/prometheus_exporter.rb +34 -0
- data/lib/pgbus/metrics/subscriber.rb +190 -0
- data/lib/pgbus/metrics.rb +42 -0
- data/lib/pgbus/process/consumer.rb +215 -8
- data/lib/pgbus/process/consumer_priority.rb +34 -0
- data/lib/pgbus/process/dispatcher.rb +265 -41
- data/lib/pgbus/process/heartbeat.rb +18 -5
- data/lib/pgbus/process/memory_usage.rb +48 -0
- data/lib/pgbus/process/notify_listener.rb +26 -7
- data/lib/pgbus/process/notify_probe.rb +96 -0
- data/lib/pgbus/process/primary_validator.rb +53 -0
- data/lib/pgbus/process/signal_handler.rb +6 -0
- data/lib/pgbus/process/supervisor.rb +396 -46
- data/lib/pgbus/process/worker.rb +298 -35
- data/lib/pgbus/recurring/scheduler.rb +15 -1
- data/lib/pgbus/streams/turbo_broadcastable.rb +7 -5
- data/lib/pgbus/table_maintenance.rb +13 -2
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source.rb +20 -4
- data/lib/pgbus/web/health_app.rb +102 -0
- data/lib/pgbus/web/health_server.rb +144 -0
- data/lib/pgbus/web/streamer/instance.rb +55 -1
- data/lib/pgbus/web/streamer/listener.rb +72 -9
- data/lib/pgbus.rb +37 -0
- data/lib/rubocop/cop/pgbus/no_ruby_timeout.rb +42 -0
- data/lib/rubocop/pgbus.rb +5 -0
- data/lib/tasks/pgbus_doctor.rake +12 -0
- metadata +18 -1
data/lib/pgbus/config_loader.rb
CHANGED
|
@@ -11,16 +11,35 @@ module Pgbus
|
|
|
11
11
|
env ||= (defined?(Rails) && Rails.respond_to?(:env) && Rails.env) || ENV.fetch("PGBUS_ENV", "development")
|
|
12
12
|
raw = File.read(path)
|
|
13
13
|
parsed = YAML.safe_load(ERB.new(raw).result, permitted_classes: [Symbol], aliases: true)
|
|
14
|
-
|
|
15
|
-
|
|
14
|
+
# Distinguish sectioned (top-level env keys mapping to Hashes) from
|
|
15
|
+
# flat (top-level setter keys mapping to scalars/arrays). parsed.key?(env)
|
|
16
|
+
# alone can't tell them apart, so a flat file silently lost its typo
|
|
17
|
+
# warnings whenever no env section happened to match its keys.
|
|
18
|
+
if sectioned?(parsed)
|
|
19
|
+
apply(parsed.fetch(env)) if parsed.key?(env)
|
|
20
|
+
else
|
|
21
|
+
apply(parsed)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def sectioned?(parsed)
|
|
26
|
+
parsed.is_a?(Hash) && parsed.any? && parsed.values.all?(Hash)
|
|
16
27
|
end
|
|
17
28
|
|
|
18
|
-
def apply(hash)
|
|
29
|
+
def apply(hash, warn_unknown: true)
|
|
19
30
|
config = Pgbus.configuration
|
|
20
31
|
hash.each do |key, value|
|
|
21
32
|
setter = :"#{key}="
|
|
22
|
-
|
|
33
|
+
if config.respond_to?(setter)
|
|
34
|
+
config.public_send(setter, value)
|
|
35
|
+
elsif warn_unknown
|
|
36
|
+
config.logger.warn { "[Pgbus] Unknown configuration key ignored: #{key.inspect} — check for typos in pgbus.yml" }
|
|
37
|
+
end
|
|
23
38
|
end
|
|
39
|
+
# Validate eagerly so a bad YAML value aborts boot with an ArgumentError
|
|
40
|
+
# naming the offending key, instead of failing later in a worker path.
|
|
41
|
+
# Honors an `eager_validation: false` key in the same hash (applied above).
|
|
42
|
+
config.validate! if config.eager_validation
|
|
24
43
|
config
|
|
25
44
|
end
|
|
26
45
|
end
|
data/lib/pgbus/configuration.rb
CHANGED
|
@@ -86,7 +86,7 @@ module Pgbus
|
|
|
86
86
|
attr_reader :pgmq_schema_mode
|
|
87
87
|
|
|
88
88
|
# Event consumers
|
|
89
|
-
|
|
89
|
+
attr_reader :event_consumers
|
|
90
90
|
|
|
91
91
|
# Recurring jobs
|
|
92
92
|
attr_accessor :recurring_tasks, :recurring_schedule_interval, :recurring_tasks_file, :skip_recurring
|
|
@@ -103,7 +103,7 @@ module Pgbus
|
|
|
103
103
|
attr_accessor :zombie_detection
|
|
104
104
|
|
|
105
105
|
# Job stats
|
|
106
|
-
attr_accessor :stats_enabled
|
|
106
|
+
attr_accessor :stats_enabled, :stats_flush_size, :stats_flush_interval
|
|
107
107
|
attr_reader :stats_retention # rubocop:disable Style/AccessorGrouping
|
|
108
108
|
|
|
109
109
|
# Web dashboard
|
|
@@ -112,6 +112,13 @@ module Pgbus
|
|
|
112
112
|
:metrics_enabled,
|
|
113
113
|
:dashboard_filter_parameters, :dashboard_filter_sensitive
|
|
114
114
|
|
|
115
|
+
# HTTP health endpoints (liveness/readiness for orchestrators like
|
|
116
|
+
# Kubernetes). `health_port` nil (default) leaves the standalone server in
|
|
117
|
+
# the supervisor disabled — mount Pgbus::Web::HealthApp in the host Rails
|
|
118
|
+
# app instead. When set, Supervisor#run serves /livez and /readyz over a
|
|
119
|
+
# plain TCPServer bound to `health_bind` (default localhost).
|
|
120
|
+
attr_accessor :health_port, :health_bind
|
|
121
|
+
|
|
115
122
|
# Streams (turbo-rails replacement, SSE-based)
|
|
116
123
|
attr_accessor :streams_enabled, :streams_path, :streams_queue_prefix, :streams_signed_name_secret,
|
|
117
124
|
:streams_default_retention, :streams_retention, :streams_heartbeat_interval,
|
|
@@ -136,6 +143,21 @@ module Pgbus
|
|
|
136
143
|
# Set to false to opt out without uninstalling the appsignal gem.
|
|
137
144
|
attr_accessor :appsignal_enabled, :appsignal_probe_enabled
|
|
138
145
|
|
|
146
|
+
# Generic metrics adapter (Pgbus::Metrics). Consumes the same pgbus.*
|
|
147
|
+
# instrumentation events as AppSignal and forwards them to a backend:
|
|
148
|
+
# nil (default) — install nothing, zero overhead
|
|
149
|
+
# :prometheus — in-process registry, scraped via PrometheusExporter
|
|
150
|
+
# :statsd — UDP datagrams to statsd_host:statsd_port
|
|
151
|
+
# a Pgbus::Metrics::Backend instance — a custom backend
|
|
152
|
+
attr_accessor :metrics_backend, :statsd_host, :statsd_port
|
|
153
|
+
|
|
154
|
+
# When true (default), Pgbus.configure and ConfigLoader.apply run
|
|
155
|
+
# validate! after applying settings, so an invalid value fails loud at
|
|
156
|
+
# boot instead of dormant until a worker path consumes it. Set false to
|
|
157
|
+
# opt out (exotic setups that intentionally hold a transiently-invalid
|
|
158
|
+
# config); explicit validate! still works.
|
|
159
|
+
attr_accessor :eager_validation
|
|
160
|
+
|
|
139
161
|
def initialize
|
|
140
162
|
@database_url = nil
|
|
141
163
|
@connection_params = nil
|
|
@@ -209,6 +231,11 @@ module Pgbus
|
|
|
209
231
|
|
|
210
232
|
@stats_enabled = true
|
|
211
233
|
@stats_retention = 30 * 24 * 3600 # 30 days
|
|
234
|
+
# StatBuffer flush thresholds. Buffered stats are lost on SIGKILL, so a
|
|
235
|
+
# smaller size/interval shrinks the residual loss window at the cost of
|
|
236
|
+
# more frequent bulk inserts. Defaults mirror StatBuffer's own constants.
|
|
237
|
+
@stats_flush_size = StatBuffer::DEFAULT_FLUSH_SIZE
|
|
238
|
+
@stats_flush_interval = StatBuffer::DEFAULT_FLUSH_INTERVAL
|
|
212
239
|
|
|
213
240
|
@connects_to = nil
|
|
214
241
|
|
|
@@ -224,6 +251,10 @@ module Pgbus
|
|
|
224
251
|
@dashboard_filter_parameters = nil # nil = auto-detect from Rails, then fall back to defaults
|
|
225
252
|
@dashboard_filter_sensitive = true
|
|
226
253
|
|
|
254
|
+
# HTTP health endpoints — nil port disables the standalone server.
|
|
255
|
+
@health_port = nil
|
|
256
|
+
@health_bind = "127.0.0.1"
|
|
257
|
+
|
|
227
258
|
@streams_enabled = true
|
|
228
259
|
@streams_path = nil
|
|
229
260
|
@streams_queue_prefix = "pgbus_stream"
|
|
@@ -287,6 +318,13 @@ module Pgbus
|
|
|
287
318
|
# the same process, so the operator can disable it independently.
|
|
288
319
|
@appsignal_enabled = true
|
|
289
320
|
@appsignal_probe_enabled = true
|
|
321
|
+
|
|
322
|
+
# Generic metrics adapter: off by default (no subscription installed).
|
|
323
|
+
@metrics_backend = nil
|
|
324
|
+
@statsd_host = "127.0.0.1"
|
|
325
|
+
@statsd_port = 8125
|
|
326
|
+
|
|
327
|
+
@eager_validation = true
|
|
290
328
|
end
|
|
291
329
|
|
|
292
330
|
def queue_name(name)
|
|
@@ -311,7 +349,7 @@ module Pgbus
|
|
|
311
349
|
# Returns the execution mode for a specific worker config hash,
|
|
312
350
|
# falling back to the global execution_mode setting.
|
|
313
351
|
def execution_mode_for(worker_config)
|
|
314
|
-
mode = worker_config[:execution_mode] ||
|
|
352
|
+
mode = worker_config[:execution_mode] || execution_mode
|
|
315
353
|
ExecutionPools.normalize_mode(mode)
|
|
316
354
|
end
|
|
317
355
|
|
|
@@ -393,15 +431,26 @@ module Pgbus
|
|
|
393
431
|
raise ArgumentError, "read_timeout must be a positive number or nil to disable"
|
|
394
432
|
end
|
|
395
433
|
|
|
434
|
+
unless stats_flush_size.is_a?(Integer) && stats_flush_size.positive?
|
|
435
|
+
raise ArgumentError, "stats_flush_size must be a positive integer"
|
|
436
|
+
end
|
|
437
|
+
unless stats_flush_interval.is_a?(Numeric) && stats_flush_interval.positive?
|
|
438
|
+
raise ArgumentError, "stats_flush_interval must be a positive number"
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
unless health_port.nil? || (health_port.is_a?(Integer) && health_port.between?(1, 65_535))
|
|
442
|
+
raise ArgumentError, "health_port must be an integer between 1 and 65535 or nil to disable"
|
|
443
|
+
end
|
|
444
|
+
|
|
396
445
|
# Validate global execution_mode
|
|
397
446
|
ExecutionPools.normalize_mode(execution_mode)
|
|
398
447
|
|
|
399
448
|
Array(workers).each do |w|
|
|
400
|
-
threads = w[:threads] ||
|
|
449
|
+
threads = w[:threads] || 5
|
|
401
450
|
raise ArgumentError, "worker threads must be > 0" unless threads.is_a?(Integer) && threads.positive?
|
|
402
451
|
|
|
403
452
|
# Validate per-worker execution_mode override if present
|
|
404
|
-
mode = w[:execution_mode]
|
|
453
|
+
mode = w[:execution_mode]
|
|
405
454
|
ExecutionPools.normalize_mode(mode) if mode
|
|
406
455
|
end
|
|
407
456
|
|
|
@@ -416,6 +465,7 @@ module Pgbus
|
|
|
416
465
|
end
|
|
417
466
|
|
|
418
467
|
validate_streams!
|
|
468
|
+
validate_metrics_backend!
|
|
419
469
|
|
|
420
470
|
self
|
|
421
471
|
end
|
|
@@ -465,6 +515,17 @@ module Pgbus
|
|
|
465
515
|
raise ArgumentError, "streams_orphan_threshold must be a positive number or nil to disable"
|
|
466
516
|
end
|
|
467
517
|
|
|
518
|
+
def validate_metrics_backend!
|
|
519
|
+
value = metrics_backend
|
|
520
|
+
return if value.nil?
|
|
521
|
+
return if %i[prometheus statsd].include?(value)
|
|
522
|
+
return if defined?(Pgbus::Metrics::Backend) && value.is_a?(Pgbus::Metrics::Backend)
|
|
523
|
+
|
|
524
|
+
raise ArgumentError,
|
|
525
|
+
"metrics_backend must be nil, :prometheus, :statsd, or a " \
|
|
526
|
+
"Pgbus::Metrics::Backend instance (got #{value.inspect})"
|
|
527
|
+
end
|
|
528
|
+
|
|
468
529
|
# Returns true if the given stream name should be durable based on
|
|
469
530
|
# `streams_durable_patterns` (exact string or Regexp match) or the
|
|
470
531
|
# `streams_default_broadcast_mode` fallback.
|
|
@@ -552,13 +613,29 @@ module Pgbus
|
|
|
552
613
|
parsed = CapsuleDSL.parse(value)
|
|
553
614
|
assign_auto_names(parsed)
|
|
554
615
|
when Array
|
|
555
|
-
value
|
|
616
|
+
value.map { |entry| normalize_entry(entry, group: "worker") }
|
|
556
617
|
else
|
|
557
618
|
raise ArgumentError,
|
|
558
619
|
"workers must be a String (DSL), Array (legacy form), or nil — got #{value.class}"
|
|
559
620
|
end
|
|
560
621
|
end
|
|
561
622
|
|
|
623
|
+
# Event consumer configs arrive with symbol keys (Ruby DSL) or string keys
|
|
624
|
+
# (YAML via ConfigLoader). Normalize once here so every read site uses
|
|
625
|
+
# symbol access only. nil passes through (no consumers configured).
|
|
626
|
+
def event_consumers=(value)
|
|
627
|
+
@event_consumers =
|
|
628
|
+
case value
|
|
629
|
+
when nil
|
|
630
|
+
nil
|
|
631
|
+
when Array
|
|
632
|
+
value.map { |entry| normalize_entry(entry, group: "event_consumer") }
|
|
633
|
+
else
|
|
634
|
+
raise ArgumentError,
|
|
635
|
+
"event_consumers must be an Array or nil — got #{value.class}"
|
|
636
|
+
end
|
|
637
|
+
end
|
|
638
|
+
|
|
562
639
|
# Define a named capsule and append it to the workers list.
|
|
563
640
|
#
|
|
564
641
|
# c.capsule :critical, queues: %w[critical], threads: 5
|
|
@@ -850,11 +927,20 @@ module Pgbus
|
|
|
850
927
|
numeric
|
|
851
928
|
end
|
|
852
929
|
|
|
853
|
-
#
|
|
854
|
-
#
|
|
930
|
+
# Symbolize the top-level keys of a worker/consumer config entry so every
|
|
931
|
+
# downstream read site can use symbol access only. YAML (via ConfigLoader)
|
|
932
|
+
# yields string keys; the Ruby DSL and +capsule+ builder yield symbols.
|
|
933
|
+
# Entries are flat hashes with array/scalar values — no deep recursion.
|
|
934
|
+
def normalize_entry(entry, group:)
|
|
935
|
+
raise ArgumentError, "#{group} entry must be a Hash, got #{entry.class}: #{entry.inspect}" unless entry.is_a?(Hash)
|
|
936
|
+
|
|
937
|
+
entry.transform_keys(&:to_sym)
|
|
938
|
+
end
|
|
939
|
+
|
|
940
|
+
# Read a capsule's name (symbol key after normalization), as a string for
|
|
941
|
+
# comparison. Returns nil for unnamed (legacy) entries.
|
|
855
942
|
def capsule_name(entry)
|
|
856
|
-
|
|
857
|
-
raw&.to_s
|
|
943
|
+
entry[:name]&.to_s
|
|
858
944
|
end
|
|
859
945
|
|
|
860
946
|
# Auto-assign :name to parsed capsules where the first queue token would
|
|
@@ -887,7 +973,7 @@ module Pgbus
|
|
|
887
973
|
existing_named = (@workers || []).select { |c| capsule_name(c) }
|
|
888
974
|
return if existing_named.empty?
|
|
889
975
|
|
|
890
|
-
existing_queues = existing_named.flat_map { |c| c[:queues] ||
|
|
976
|
+
existing_queues = existing_named.flat_map { |c| c[:queues] || [] }
|
|
891
977
|
return if existing_queues.empty?
|
|
892
978
|
|
|
893
979
|
if existing_queues.include?(CapsuleDSL::WILDCARD)
|
|
@@ -915,7 +1001,7 @@ module Pgbus
|
|
|
915
1001
|
return 0 unless entries
|
|
916
1002
|
|
|
917
1003
|
entries.sum do |entry|
|
|
918
|
-
threads = entry[:threads] ||
|
|
1004
|
+
threads = entry[:threads] || default_threads
|
|
919
1005
|
unless threads.is_a?(Integer) && threads.positive?
|
|
920
1006
|
raise ArgumentError,
|
|
921
1007
|
"#{group} threads must be a positive integer, got #{threads.inspect}"
|
data/lib/pgbus/dedup_cache.rb
CHANGED
|
@@ -60,11 +60,19 @@ module Pgbus
|
|
|
60
60
|
|
|
61
61
|
def evict(key)
|
|
62
62
|
@cache.delete(key)
|
|
63
|
+
# Concurrent::Array#delete is O(n), but this only runs on expiry and n is
|
|
64
|
+
# bounded by @max_size, so it stays cheap. Without it, @insertion_order
|
|
65
|
+
# leaks and stale duplicates can evict live keys in evict_oldest.
|
|
66
|
+
@insertion_order.delete(key)
|
|
63
67
|
end
|
|
64
68
|
|
|
65
69
|
def evict_oldest
|
|
66
70
|
while @cache.size >= @max_size && !@insertion_order.empty?
|
|
67
71
|
oldest_key = @insertion_order.shift
|
|
72
|
+
# Skip residual order entries for keys already gone from @cache so a
|
|
73
|
+
# stale duplicate never evicts a freshly re-marked (live) key.
|
|
74
|
+
next unless @cache.key?(oldest_key)
|
|
75
|
+
|
|
68
76
|
@cache.delete(oldest_key)
|
|
69
77
|
end
|
|
70
78
|
end
|
data/lib/pgbus/doctor.rb
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "uri"
|
|
4
|
+
require "pgbus/mcp/health_analyzer"
|
|
5
|
+
|
|
6
|
+
module Pgbus
|
|
7
|
+
# Preflight diagnostics for a pgbus deployment — the single command that
|
|
8
|
+
# answers "is this environment healthy enough to run?". Runs six checks and
|
|
9
|
+
# returns a machine-readable result plus a human report, so `pgbus doctor`
|
|
10
|
+
# and `rake pgbus:doctor` can gate a deploy or CI run (exit 0 on success,
|
|
11
|
+
# 1 on any failure).
|
|
12
|
+
#
|
|
13
|
+
# Doctor never touches PGMQ or PostgreSQL directly: every probe goes through
|
|
14
|
+
# Pgbus::Client (DB, PGMQ schema, queues, NOTIFY) or Pgbus::Web::DataSource
|
|
15
|
+
# (process liveness, via Pgbus::MCP::HealthAnalyzer). It also never raises —
|
|
16
|
+
# a broken environment turns into :fail results, never a crash — so it is
|
|
17
|
+
# safe to run against a database that is down or a half-installed schema.
|
|
18
|
+
class Doctor
|
|
19
|
+
# A single check result. status is one of :ok, :warn, :fail.
|
|
20
|
+
Check = Struct.new(:name, :status, :detail) do
|
|
21
|
+
def to_h
|
|
22
|
+
{ name: name, status: status, detail: detail }
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
STATUS_ICON = { ok: "✓", warn: "!", fail: "✗" }.freeze
|
|
27
|
+
|
|
28
|
+
def initialize(config: Pgbus.configuration, client: Pgbus.client, data_source: nil)
|
|
29
|
+
@config = config
|
|
30
|
+
@client = client
|
|
31
|
+
@data_source = data_source || Pgbus::Web::DataSource.new(client: client)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Run all checks and return an array of result hashes:
|
|
35
|
+
# { name:, status: :ok|:warn|:fail, detail: }
|
|
36
|
+
def run
|
|
37
|
+
@run ||= [
|
|
38
|
+
check_configuration,
|
|
39
|
+
check_database,
|
|
40
|
+
check_pgmq_schema,
|
|
41
|
+
check_queues,
|
|
42
|
+
check_notify,
|
|
43
|
+
check_processes
|
|
44
|
+
].map(&:to_h)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# True when no check failed. Warnings do not fail the run — they surface a
|
|
48
|
+
# concern (e.g. an outdated PGMQ schema) without blocking a deploy.
|
|
49
|
+
def success?
|
|
50
|
+
run.none? { |c| c[:status] == :fail }
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Human-readable report: one line per check, then a resolved-config summary
|
|
54
|
+
# with passwords redacted. Suitable for stdout in the CLI and rake task.
|
|
55
|
+
def report
|
|
56
|
+
lines = ["Pgbus Doctor", "=" * 40]
|
|
57
|
+
run.each do |check|
|
|
58
|
+
icon = STATUS_ICON.fetch(check[:status], "?")
|
|
59
|
+
lines << format("%<icon>s %<name>-22s %<detail>s", icon: icon, name: check[:name], detail: check[:detail])
|
|
60
|
+
end
|
|
61
|
+
lines << ""
|
|
62
|
+
lines << "Configuration"
|
|
63
|
+
lines << ("-" * 40)
|
|
64
|
+
config_summary.each { |key, value| lines << format(" %<key>-20s %<value>s", key: key, value: value) }
|
|
65
|
+
lines.join("\n")
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Resolved-config summary for the report. Password-bearing fields
|
|
69
|
+
# (database_url, connection_params) are redacted.
|
|
70
|
+
def config_summary
|
|
71
|
+
{
|
|
72
|
+
queue_prefix: @config.queue_prefix,
|
|
73
|
+
default_queue: @config.default_queue,
|
|
74
|
+
pgmq_schema_mode: @config.pgmq_schema_mode,
|
|
75
|
+
resolved_pool_size: safe { @config.resolved_pool_size },
|
|
76
|
+
listen_notify: @config.listen_notify,
|
|
77
|
+
roles: @config.roles || "all",
|
|
78
|
+
capsules: capsule_summary,
|
|
79
|
+
database_url: redacted_database_url,
|
|
80
|
+
connection_params: redacted_connection_params
|
|
81
|
+
}.compact
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
private
|
|
85
|
+
|
|
86
|
+
# 1. Configuration validity.
|
|
87
|
+
def check_configuration
|
|
88
|
+
@config.validate!
|
|
89
|
+
Check.new(name: "Configuration", status: :ok, detail: "valid")
|
|
90
|
+
rescue ArgumentError => e
|
|
91
|
+
Check.new(name: "Configuration", status: :fail, detail: e.message)
|
|
92
|
+
rescue StandardError => e
|
|
93
|
+
Check.new(name: "Configuration", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# 2. Database connectivity — SELECT 1 via Client#ping.
|
|
97
|
+
def check_database
|
|
98
|
+
@client.ping
|
|
99
|
+
Check.new(name: "Database", status: :ok, detail: "reachable")
|
|
100
|
+
rescue StandardError => e
|
|
101
|
+
Check.new(name: "Database", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# 3. PGMQ schema presence + installed-vs-vendored version.
|
|
105
|
+
#
|
|
106
|
+
# A nil version does NOT automatically mean "not installed": PGMQ installed
|
|
107
|
+
# via the extension, or before pgbus added version tracking, leaves the
|
|
108
|
+
# pgmq schema fully working with no row in pgbus_pgmq_schema_versions. So we
|
|
109
|
+
# distinguish, mirroring `pgbus:pgmq:status`: schema absent → fail; schema
|
|
110
|
+
# present but untracked → warn; tracked but behind → warn; current → ok.
|
|
111
|
+
def check_pgmq_schema
|
|
112
|
+
installed = @client.pgmq_schema_version
|
|
113
|
+
latest = Pgbus::PgmqSchema.latest_version
|
|
114
|
+
|
|
115
|
+
if installed.nil?
|
|
116
|
+
unless @client.pgmq_installed?
|
|
117
|
+
return Check.new(name: "PGMQ schema", status: :fail,
|
|
118
|
+
detail: "not installed — run `rails generate pgbus:install`")
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
return Check.new(name: "PGMQ schema", status: :warn,
|
|
122
|
+
detail: "installed but no version tracking — run `rails generate pgbus:upgrade_pgmq`")
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
if Gem::Version.new(installed) < Gem::Version.new(latest)
|
|
126
|
+
Check.new(name: "PGMQ schema", status: :warn,
|
|
127
|
+
detail: "installed #{installed}, vendored #{latest} — run `rails generate pgbus:upgrade_pgmq`")
|
|
128
|
+
else
|
|
129
|
+
Check.new(name: "PGMQ schema", status: :ok, detail: "up to date (#{installed})")
|
|
130
|
+
end
|
|
131
|
+
rescue StandardError => e
|
|
132
|
+
Check.new(name: "PGMQ schema", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# 4. Queue existence — every configured queue must have its PGMQ table(s).
|
|
136
|
+
# Resolve each logical queue to the PHYSICAL names bootstrap creates (via the
|
|
137
|
+
# client's queue strategy) so a priority queue's _p0.._pN sub-tables are what
|
|
138
|
+
# we diff against list_queues — not the bare prefixed name priority mode
|
|
139
|
+
# never creates.
|
|
140
|
+
def check_queues
|
|
141
|
+
configured = @client.configured_queues.flat_map { |q| @client.physical_queue_names(q) }
|
|
142
|
+
existing = existing_queue_names
|
|
143
|
+
missing = configured - existing
|
|
144
|
+
|
|
145
|
+
if missing.empty?
|
|
146
|
+
Check.new(name: "Queues", status: :ok, detail: "#{configured.size} configured queue(s) present")
|
|
147
|
+
else
|
|
148
|
+
Check.new(name: "Queues", status: :fail, detail: "missing queue(s): #{missing.join(", ")}")
|
|
149
|
+
end
|
|
150
|
+
rescue StandardError => e
|
|
151
|
+
Check.new(name: "Queues", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# 5. LISTEN/NOTIFY — when configured on, every configured queue should carry
|
|
155
|
+
# the insert-NOTIFY trigger. A missing trigger is a warning, not a failure:
|
|
156
|
+
# pgbus still works (it falls back to polling), it just loses instant wakeup.
|
|
157
|
+
def check_notify
|
|
158
|
+
return Check.new(name: "LISTEN/NOTIFY", status: :ok, detail: "disabled in config") unless @config.listen_notify
|
|
159
|
+
|
|
160
|
+
without_trigger = @client.configured_queues.reject { |q| @client.notify_enabled?(q) }
|
|
161
|
+
if without_trigger.empty?
|
|
162
|
+
Check.new(name: "LISTEN/NOTIFY", status: :ok, detail: "triggers live on all configured queues")
|
|
163
|
+
else
|
|
164
|
+
Check.new(name: "LISTEN/NOTIFY", status: :warn,
|
|
165
|
+
detail: "no insert trigger on: #{without_trigger.join(", ")} (falling back to polling)")
|
|
166
|
+
end
|
|
167
|
+
rescue StandardError => e
|
|
168
|
+
Check.new(name: "LISTEN/NOTIFY", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# 6. Process liveness — the top-level health verdict (OK/DEGRADED/STALLED)
|
|
172
|
+
# from the shared HealthAnalyzer. STALLED (the silent-worker-wedge) is a
|
|
173
|
+
# failure; DEGRADED is a warning; OK passes.
|
|
174
|
+
def check_processes
|
|
175
|
+
verdict = Pgbus::MCP::HealthAnalyzer.new(@data_source).verdict
|
|
176
|
+
status = verdict[:status]
|
|
177
|
+
detail = Array(verdict[:reasons]).first || status
|
|
178
|
+
|
|
179
|
+
case status
|
|
180
|
+
when "STALLED" then Check.new(name: "Process liveness", status: :fail, detail: detail)
|
|
181
|
+
when "DEGRADED" then Check.new(name: "Process liveness", status: :warn, detail: detail)
|
|
182
|
+
else Check.new(name: "Process liveness", status: :ok, detail: "OK")
|
|
183
|
+
end
|
|
184
|
+
rescue StandardError => e
|
|
185
|
+
Check.new(name: "Process liveness", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# Physical queue names known to PGMQ. list_queues rows may be Hashes (with a
|
|
189
|
+
# :queue_name key) or value objects responding to #queue_name.
|
|
190
|
+
def existing_queue_names
|
|
191
|
+
Array(@client.list_queues).map do |row|
|
|
192
|
+
if row.respond_to?(:queue_name)
|
|
193
|
+
row.queue_name
|
|
194
|
+
elsif row.is_a?(Hash)
|
|
195
|
+
row[:queue_name] || row["queue_name"]
|
|
196
|
+
end
|
|
197
|
+
end.compact
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def capsule_summary
|
|
201
|
+
Array(@config.workers).map { |w| w[:name] || (w[:queues] || []).join("+") }.join(", ")
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# Redact the password from a libpq URL/URI while keeping host/db visible so
|
|
205
|
+
# the operator can still confirm which database they pointed at.
|
|
206
|
+
def redacted_database_url
|
|
207
|
+
return nil unless @config.database_url
|
|
208
|
+
|
|
209
|
+
redact_url(@config.database_url)
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
# connection_params is a free-form libpq keyword hash, so redact every key
|
|
213
|
+
# whose name looks secret (password, sslpassword, ...) — not just :password —
|
|
214
|
+
# rather than assuming a fixed shape. Preserves symbol/string key form.
|
|
215
|
+
SECRET_KEY_PATTERN = /pass|secret|token/i
|
|
216
|
+
private_constant :SECRET_KEY_PATTERN
|
|
217
|
+
|
|
218
|
+
def redacted_connection_params
|
|
219
|
+
params = @config.connection_params
|
|
220
|
+
return nil unless params.is_a?(Hash)
|
|
221
|
+
|
|
222
|
+
params.each_with_object({}) do |(key, value), out|
|
|
223
|
+
out[key] = key.to_s.match?(SECRET_KEY_PATTERN) ? "[REDACTED]" : value
|
|
224
|
+
end
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# Replace the password in a userinfo authority (scheme://user:pass@host) or a
|
|
228
|
+
# key=value conninfo string (password=secret). Falls back to a blanket
|
|
229
|
+
# redaction label if parsing fails, never leaking the original.
|
|
230
|
+
#
|
|
231
|
+
# The userinfo password can itself contain '@' and ':' (common in generated
|
|
232
|
+
# secrets), so match it greedily up to the LAST '@' before the authority's
|
|
233
|
+
# host — a lazy `[^@]+` would stop at the first '@' and leak the remainder.
|
|
234
|
+
def redact_url(url)
|
|
235
|
+
redacted = url.sub(%r{(://[^:/@]+:).+(@[^@/]*(?:[/?]|\z))}, '\1[REDACTED]\2')
|
|
236
|
+
redacted.gsub(/(\bpassword=)('[^']*'|[^\s'"]+)/i, '\1[REDACTED]')
|
|
237
|
+
rescue StandardError
|
|
238
|
+
"[REDACTED]"
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
# Wrap a value-producing block so a broken resolver (e.g. resolved_pool_size
|
|
242
|
+
# raising on a malformed worker config) degrades to nil instead of raising
|
|
243
|
+
# out of config_summary.
|
|
244
|
+
def safe
|
|
245
|
+
yield
|
|
246
|
+
rescue StandardError
|
|
247
|
+
nil
|
|
248
|
+
end
|
|
249
|
+
end
|
|
250
|
+
end
|
data/lib/pgbus/engine.rb
CHANGED
|
@@ -60,6 +60,7 @@ module Pgbus
|
|
|
60
60
|
load File.expand_path("../tasks/pgbus_pgmq.rake", __dir__)
|
|
61
61
|
load File.expand_path("../tasks/pgbus_streams.rake", __dir__)
|
|
62
62
|
load File.expand_path("../tasks/pgbus_autovacuum.rake", __dir__)
|
|
63
|
+
load File.expand_path("../tasks/pgbus_doctor.rake", __dir__)
|
|
63
64
|
end
|
|
64
65
|
|
|
65
66
|
initializer "pgbus.i18n" do
|
|
@@ -86,6 +87,20 @@ module Pgbus
|
|
|
86
87
|
end
|
|
87
88
|
end
|
|
88
89
|
|
|
90
|
+
# Generic metrics adapter. Opt-in via config.metrics_backend; nil (default)
|
|
91
|
+
# installs no subscription, so there is zero overhead when unused. Runs
|
|
92
|
+
# independently of AppSignal — both consume the same pgbus.* events and both
|
|
93
|
+
# can be active at once.
|
|
94
|
+
initializer "pgbus.metrics", after: :load_config_initializers do
|
|
95
|
+
ActiveSupport.on_load(:after_initialize) do
|
|
96
|
+
next if Pgbus.configuration.metrics_backend.nil?
|
|
97
|
+
|
|
98
|
+
backend = Pgbus::Metrics.resolve_backend(Pgbus.configuration.metrics_backend)
|
|
99
|
+
Pgbus::Metrics::Subscriber.install!(backend: backend)
|
|
100
|
+
Pgbus.logger.info { "[Pgbus] Metrics subscriber installed (#{backend.class})" }
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
89
104
|
# Install the watermark cache middleware ahead of the app's own
|
|
90
105
|
# middleware so the thread-local cache is cleared between every
|
|
91
106
|
# Rack request. Without this, repeated page renders served by the
|
|
@@ -46,6 +46,13 @@ module Pgbus
|
|
|
46
46
|
available_capacity.positive?
|
|
47
47
|
end
|
|
48
48
|
|
|
49
|
+
# True only when every slot is free — all in-flight work has finished.
|
|
50
|
+
# idle? answers "can this pool accept work?"; quiesced? answers
|
|
51
|
+
# "has this pool drained?". The worker drain loop needs the latter.
|
|
52
|
+
def quiesced?
|
|
53
|
+
available_capacity >= @capacity
|
|
54
|
+
end
|
|
55
|
+
|
|
49
56
|
def shutdown
|
|
50
57
|
@state_mutex.synchronize do
|
|
51
58
|
return false if @shutdown_flag
|
|
@@ -37,6 +37,13 @@ module Pgbus
|
|
|
37
37
|
available_capacity.positive?
|
|
38
38
|
end
|
|
39
39
|
|
|
40
|
+
# True only when every slot is free — all in-flight work has finished.
|
|
41
|
+
# idle? answers "can this pool accept work?"; quiesced? answers
|
|
42
|
+
# "has this pool drained?". The worker drain loop needs the latter.
|
|
43
|
+
def quiesced?
|
|
44
|
+
available_capacity >= @capacity
|
|
45
|
+
end
|
|
46
|
+
|
|
40
47
|
def shutdown
|
|
41
48
|
@pool.shutdown
|
|
42
49
|
end
|
|
@@ -11,6 +11,7 @@ module Pgbus
|
|
|
11
11
|
# pgbus.client.send_batch — batch enqueue
|
|
12
12
|
# pgbus.client.read_batch — batch dequeue
|
|
13
13
|
# pgbus.client.read_message — single message dequeue
|
|
14
|
+
# pgbus.client.pool — connection pool snapshot (size/available/pool_timeout); emitted per worker heartbeat
|
|
14
15
|
# pgbus.executor.execute — full job execution (deserialize + perform + archive)
|
|
15
16
|
# pgbus.job_completed — job archived successfully
|
|
16
17
|
# pgbus.job_failed — job raised; carries :exception_object
|
|
@@ -51,8 +51,9 @@ module Pgbus
|
|
|
51
51
|
|
|
52
52
|
# The actual probe object; AppSignal calls #call once per minute.
|
|
53
53
|
class Runner
|
|
54
|
-
def initialize(data_source: nil)
|
|
54
|
+
def initialize(data_source: nil, client: nil)
|
|
55
55
|
@data_source = data_source
|
|
56
|
+
@client = client
|
|
56
57
|
@hostname = Socket.gethostname
|
|
57
58
|
end
|
|
58
59
|
|
|
@@ -63,6 +64,7 @@ module Pgbus
|
|
|
63
64
|
track_processes
|
|
64
65
|
track_summary
|
|
65
66
|
track_streams
|
|
67
|
+
track_pool
|
|
66
68
|
end
|
|
67
69
|
|
|
68
70
|
private
|
|
@@ -72,6 +74,10 @@ module Pgbus
|
|
|
72
74
|
(::Pgbus::Web::DataSource.new if defined?(::Pgbus::Web::DataSource))
|
|
73
75
|
end
|
|
74
76
|
|
|
77
|
+
def client
|
|
78
|
+
@client ||= (::Pgbus.client if defined?(::Pgbus) && ::Pgbus.respond_to?(:client))
|
|
79
|
+
end
|
|
80
|
+
|
|
75
81
|
def track_queues
|
|
76
82
|
data_source.queues_with_metrics.each do |q|
|
|
77
83
|
tags = { queue: q[:name] }
|
|
@@ -110,6 +116,22 @@ module Pgbus
|
|
|
110
116
|
log_failure("summary metrics", e)
|
|
111
117
|
end
|
|
112
118
|
|
|
119
|
+
# The PGMQ connection pool is per-process (each host owns its own
|
|
120
|
+
# pool), so — unlike the cluster-wide queue/summary gauges — these
|
|
121
|
+
# are tagged with the hostname, same policy as active_processes.
|
|
122
|
+
# pool_stats already rescues to {} internally; the outer rescue here
|
|
123
|
+
# keeps a probe iteration alive if the client itself is unavailable.
|
|
124
|
+
def track_pool
|
|
125
|
+
stats = client&.pool_stats || {}
|
|
126
|
+
return if stats.empty?
|
|
127
|
+
|
|
128
|
+
tags = { hostname: @hostname }
|
|
129
|
+
gauge "pool_size", stats[:size], tags
|
|
130
|
+
gauge "pool_available", stats[:available], tags
|
|
131
|
+
rescue StandardError => e
|
|
132
|
+
log_failure("pool metrics", e)
|
|
133
|
+
end
|
|
134
|
+
|
|
113
135
|
def track_streams
|
|
114
136
|
return unless data_source.respond_to?(:stream_stats_available?) &&
|
|
115
137
|
data_source.stream_stats_available?
|