pgbus 0.9.7 → 0.9.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +38 -0
  3. data/README.md +119 -1
  4. data/Rakefile +10 -1
  5. data/app/helpers/pgbus/application_helper.rb +37 -0
  6. data/app/views/pgbus/processes/_processes_table.html.erb +4 -1
  7. data/config/locales/da.yml +4 -0
  8. data/config/locales/de.yml +4 -0
  9. data/config/locales/en.yml +4 -0
  10. data/config/locales/es.yml +4 -0
  11. data/config/locales/fi.yml +4 -0
  12. data/config/locales/fr.yml +4 -0
  13. data/config/locales/it.yml +4 -0
  14. data/config/locales/ja.yml +4 -0
  15. data/config/locales/nb.yml +4 -0
  16. data/config/locales/nl.yml +4 -0
  17. data/config/locales/pt.yml +4 -0
  18. data/config/locales/sv.yml +4 -0
  19. data/lib/pgbus/active_job/executor.rb +25 -4
  20. data/lib/pgbus/cli/dlq.rb +164 -0
  21. data/lib/pgbus/cli.rb +18 -1
  22. data/lib/pgbus/client/connection_health.rb +194 -0
  23. data/lib/pgbus/client.rb +592 -73
  24. data/lib/pgbus/config_loader.rb +23 -4
  25. data/lib/pgbus/configuration.rb +98 -12
  26. data/lib/pgbus/dedup_cache.rb +8 -0
  27. data/lib/pgbus/doctor.rb +250 -0
  28. data/lib/pgbus/engine.rb +15 -0
  29. data/lib/pgbus/execution_pools/async_pool.rb +7 -0
  30. data/lib/pgbus/execution_pools/thread_pool.rb +7 -0
  31. data/lib/pgbus/instrumentation.rb +1 -0
  32. data/lib/pgbus/integrations/appsignal/probe.rb +23 -1
  33. data/lib/pgbus/metrics/backend.rb +38 -0
  34. data/lib/pgbus/metrics/backends/prometheus.rb +123 -0
  35. data/lib/pgbus/metrics/backends/statsd.rb +64 -0
  36. data/lib/pgbus/metrics/prometheus_exporter.rb +34 -0
  37. data/lib/pgbus/metrics/subscriber.rb +190 -0
  38. data/lib/pgbus/metrics.rb +42 -0
  39. data/lib/pgbus/process/consumer.rb +215 -8
  40. data/lib/pgbus/process/consumer_priority.rb +34 -0
  41. data/lib/pgbus/process/dispatcher.rb +265 -41
  42. data/lib/pgbus/process/heartbeat.rb +18 -5
  43. data/lib/pgbus/process/memory_usage.rb +48 -0
  44. data/lib/pgbus/process/notify_listener.rb +26 -7
  45. data/lib/pgbus/process/notify_probe.rb +96 -0
  46. data/lib/pgbus/process/primary_validator.rb +53 -0
  47. data/lib/pgbus/process/signal_handler.rb +6 -0
  48. data/lib/pgbus/process/supervisor.rb +396 -46
  49. data/lib/pgbus/process/worker.rb +298 -35
  50. data/lib/pgbus/recurring/scheduler.rb +15 -1
  51. data/lib/pgbus/streams/turbo_broadcastable.rb +7 -5
  52. data/lib/pgbus/table_maintenance.rb +13 -2
  53. data/lib/pgbus/version.rb +1 -1
  54. data/lib/pgbus/web/data_source.rb +20 -4
  55. data/lib/pgbus/web/health_app.rb +102 -0
  56. data/lib/pgbus/web/health_server.rb +144 -0
  57. data/lib/pgbus/web/streamer/instance.rb +55 -1
  58. data/lib/pgbus/web/streamer/listener.rb +72 -9
  59. data/lib/pgbus.rb +37 -0
  60. data/lib/rubocop/cop/pgbus/no_ruby_timeout.rb +42 -0
  61. data/lib/rubocop/pgbus.rb +5 -0
  62. data/lib/tasks/pgbus_doctor.rake +12 -0
  63. metadata +18 -1
@@ -11,16 +11,35 @@ module Pgbus
11
11
  env ||= (defined?(Rails) && Rails.respond_to?(:env) && Rails.env) || ENV.fetch("PGBUS_ENV", "development")
12
12
  raw = File.read(path)
13
13
  parsed = YAML.safe_load(ERB.new(raw).result, permitted_classes: [Symbol], aliases: true)
14
- config_hash = parsed.fetch(env, parsed)
15
- apply(config_hash)
14
+ # Distinguish sectioned (top-level env keys mapping to Hashes) from
15
+ # flat (top-level setter keys mapping to scalars/arrays). parsed.key?(env)
16
+ # alone can't tell them apart, so a flat file silently lost its typo
17
+ # warnings whenever no env section happened to match its keys.
18
+ if sectioned?(parsed)
19
+ apply(parsed.fetch(env)) if parsed.key?(env)
20
+ else
21
+ apply(parsed)
22
+ end
23
+ end
24
+
25
+ def sectioned?(parsed)
26
+ parsed.is_a?(Hash) && parsed.any? && parsed.values.all?(Hash)
16
27
  end
17
28
 
18
- def apply(hash)
29
+ def apply(hash, warn_unknown: true)
19
30
  config = Pgbus.configuration
20
31
  hash.each do |key, value|
21
32
  setter = :"#{key}="
22
- config.public_send(setter, value) if config.respond_to?(setter)
33
+ if config.respond_to?(setter)
34
+ config.public_send(setter, value)
35
+ elsif warn_unknown
36
+ config.logger.warn { "[Pgbus] Unknown configuration key ignored: #{key.inspect} — check for typos in pgbus.yml" }
37
+ end
23
38
  end
39
+ # Validate eagerly so a bad YAML value aborts boot with an ArgumentError
40
+ # naming the offending key, instead of failing later in a worker path.
41
+ # Honors an `eager_validation: false` key in the same hash (applied above).
42
+ config.validate! if config.eager_validation
24
43
  config
25
44
  end
26
45
  end
@@ -86,7 +86,7 @@ module Pgbus
86
86
  attr_reader :pgmq_schema_mode
87
87
 
88
88
  # Event consumers
89
- attr_accessor :event_consumers
89
+ attr_reader :event_consumers
90
90
 
91
91
  # Recurring jobs
92
92
  attr_accessor :recurring_tasks, :recurring_schedule_interval, :recurring_tasks_file, :skip_recurring
@@ -103,7 +103,7 @@ module Pgbus
103
103
  attr_accessor :zombie_detection
104
104
 
105
105
  # Job stats
106
- attr_accessor :stats_enabled
106
+ attr_accessor :stats_enabled, :stats_flush_size, :stats_flush_interval
107
107
  attr_reader :stats_retention # rubocop:disable Style/AccessorGrouping
108
108
 
109
109
  # Web dashboard
@@ -112,6 +112,13 @@ module Pgbus
112
112
  :metrics_enabled,
113
113
  :dashboard_filter_parameters, :dashboard_filter_sensitive
114
114
 
115
+ # HTTP health endpoints (liveness/readiness for orchestrators like
116
+ # Kubernetes). `health_port` nil (default) leaves the standalone server in
117
+ # the supervisor disabled — mount Pgbus::Web::HealthApp in the host Rails
118
+ # app instead. When set, Supervisor#run serves /livez and /readyz over a
119
+ # plain TCPServer bound to `health_bind` (default localhost).
120
+ attr_accessor :health_port, :health_bind
121
+
115
122
  # Streams (turbo-rails replacement, SSE-based)
116
123
  attr_accessor :streams_enabled, :streams_path, :streams_queue_prefix, :streams_signed_name_secret,
117
124
  :streams_default_retention, :streams_retention, :streams_heartbeat_interval,
@@ -136,6 +143,21 @@ module Pgbus
136
143
  # Set to false to opt out without uninstalling the appsignal gem.
137
144
  attr_accessor :appsignal_enabled, :appsignal_probe_enabled
138
145
 
146
+ # Generic metrics adapter (Pgbus::Metrics). Consumes the same pgbus.*
147
+ # instrumentation events as AppSignal and forwards them to a backend:
148
+ # nil (default) — install nothing, zero overhead
149
+ # :prometheus — in-process registry, scraped via PrometheusExporter
150
+ # :statsd — UDP datagrams to statsd_host:statsd_port
151
+ # a Pgbus::Metrics::Backend instance — a custom backend
152
+ attr_accessor :metrics_backend, :statsd_host, :statsd_port
153
+
154
+ # When true (default), Pgbus.configure and ConfigLoader.apply run
155
+ # validate! after applying settings, so an invalid value fails loud at
156
+ # boot instead of dormant until a worker path consumes it. Set false to
157
+ # opt out (exotic setups that intentionally hold a transiently-invalid
158
+ # config); explicit validate! still works.
159
+ attr_accessor :eager_validation
160
+
139
161
  def initialize
140
162
  @database_url = nil
141
163
  @connection_params = nil
@@ -209,6 +231,11 @@ module Pgbus
209
231
 
210
232
  @stats_enabled = true
211
233
  @stats_retention = 30 * 24 * 3600 # 30 days
234
+ # StatBuffer flush thresholds. Buffered stats are lost on SIGKILL, so a
235
+ # smaller size/interval shrinks the residual loss window at the cost of
236
+ # more frequent bulk inserts. Defaults mirror StatBuffer's own constants.
237
+ @stats_flush_size = StatBuffer::DEFAULT_FLUSH_SIZE
238
+ @stats_flush_interval = StatBuffer::DEFAULT_FLUSH_INTERVAL
212
239
 
213
240
  @connects_to = nil
214
241
 
@@ -224,6 +251,10 @@ module Pgbus
224
251
  @dashboard_filter_parameters = nil # nil = auto-detect from Rails, then fall back to defaults
225
252
  @dashboard_filter_sensitive = true
226
253
 
254
+ # HTTP health endpoints — nil port disables the standalone server.
255
+ @health_port = nil
256
+ @health_bind = "127.0.0.1"
257
+
227
258
  @streams_enabled = true
228
259
  @streams_path = nil
229
260
  @streams_queue_prefix = "pgbus_stream"
@@ -287,6 +318,13 @@ module Pgbus
287
318
  # the same process, so the operator can disable it independently.
288
319
  @appsignal_enabled = true
289
320
  @appsignal_probe_enabled = true
321
+
322
+ # Generic metrics adapter: off by default (no subscription installed).
323
+ @metrics_backend = nil
324
+ @statsd_host = "127.0.0.1"
325
+ @statsd_port = 8125
326
+
327
+ @eager_validation = true
290
328
  end
291
329
 
292
330
  def queue_name(name)
@@ -311,7 +349,7 @@ module Pgbus
311
349
  # Returns the execution mode for a specific worker config hash,
312
350
  # falling back to the global execution_mode setting.
313
351
  def execution_mode_for(worker_config)
314
- mode = worker_config[:execution_mode] || worker_config["execution_mode"] || execution_mode
352
+ mode = worker_config[:execution_mode] || execution_mode
315
353
  ExecutionPools.normalize_mode(mode)
316
354
  end
317
355
 
@@ -393,15 +431,26 @@ module Pgbus
393
431
  raise ArgumentError, "read_timeout must be a positive number or nil to disable"
394
432
  end
395
433
 
434
+ unless stats_flush_size.is_a?(Integer) && stats_flush_size.positive?
435
+ raise ArgumentError, "stats_flush_size must be a positive integer"
436
+ end
437
+ unless stats_flush_interval.is_a?(Numeric) && stats_flush_interval.positive?
438
+ raise ArgumentError, "stats_flush_interval must be a positive number"
439
+ end
440
+
441
+ unless health_port.nil? || (health_port.is_a?(Integer) && health_port.between?(1, 65_535))
442
+ raise ArgumentError, "health_port must be an integer between 1 and 65535 or nil to disable"
443
+ end
444
+
396
445
  # Validate global execution_mode
397
446
  ExecutionPools.normalize_mode(execution_mode)
398
447
 
399
448
  Array(workers).each do |w|
400
- threads = w[:threads] || w["threads"] || 5
449
+ threads = w[:threads] || 5
401
450
  raise ArgumentError, "worker threads must be > 0" unless threads.is_a?(Integer) && threads.positive?
402
451
 
403
452
  # Validate per-worker execution_mode override if present
404
- mode = w[:execution_mode] || w["execution_mode"]
453
+ mode = w[:execution_mode]
405
454
  ExecutionPools.normalize_mode(mode) if mode
406
455
  end
407
456
 
@@ -416,6 +465,7 @@ module Pgbus
416
465
  end
417
466
 
418
467
  validate_streams!
468
+ validate_metrics_backend!
419
469
 
420
470
  self
421
471
  end
@@ -465,6 +515,17 @@ module Pgbus
465
515
  raise ArgumentError, "streams_orphan_threshold must be a positive number or nil to disable"
466
516
  end
467
517
 
518
+ def validate_metrics_backend!
519
+ value = metrics_backend
520
+ return if value.nil?
521
+ return if %i[prometheus statsd].include?(value)
522
+ return if defined?(Pgbus::Metrics::Backend) && value.is_a?(Pgbus::Metrics::Backend)
523
+
524
+ raise ArgumentError,
525
+ "metrics_backend must be nil, :prometheus, :statsd, or a " \
526
+ "Pgbus::Metrics::Backend instance (got #{value.inspect})"
527
+ end
528
+
468
529
  # Returns true if the given stream name should be durable based on
469
530
  # `streams_durable_patterns` (exact string or Regexp match) or the
470
531
  # `streams_default_broadcast_mode` fallback.
@@ -552,13 +613,29 @@ module Pgbus
552
613
  parsed = CapsuleDSL.parse(value)
553
614
  assign_auto_names(parsed)
554
615
  when Array
555
- value
616
+ value.map { |entry| normalize_entry(entry, group: "worker") }
556
617
  else
557
618
  raise ArgumentError,
558
619
  "workers must be a String (DSL), Array (legacy form), or nil — got #{value.class}"
559
620
  end
560
621
  end
561
622
 
623
+ # Event consumer configs arrive with symbol keys (Ruby DSL) or string keys
624
+ # (YAML via ConfigLoader). Normalize once here so every read site uses
625
+ # symbol access only. nil passes through (no consumers configured).
626
+ def event_consumers=(value)
627
+ @event_consumers =
628
+ case value
629
+ when nil
630
+ nil
631
+ when Array
632
+ value.map { |entry| normalize_entry(entry, group: "event_consumer") }
633
+ else
634
+ raise ArgumentError,
635
+ "event_consumers must be an Array or nil — got #{value.class}"
636
+ end
637
+ end
638
+
562
639
  # Define a named capsule and append it to the workers list.
563
640
  #
564
641
  # c.capsule :critical, queues: %w[critical], threads: 5
@@ -850,11 +927,20 @@ module Pgbus
850
927
  numeric
851
928
  end
852
929
 
853
- # Read a capsule's name from either symbol or string key, normalized
854
- # to a string for comparison. Returns nil for unnamed (legacy) entries.
930
+ # Symbolize the top-level keys of a worker/consumer config entry so every
931
+ # downstream read site can use symbol access only. YAML (via ConfigLoader)
932
+ # yields string keys; the Ruby DSL and +capsule+ builder yield symbols.
933
+ # Entries are flat hashes with array/scalar values — no deep recursion.
934
+ def normalize_entry(entry, group:)
935
+ raise ArgumentError, "#{group} entry must be a Hash, got #{entry.class}: #{entry.inspect}" unless entry.is_a?(Hash)
936
+
937
+ entry.transform_keys(&:to_sym)
938
+ end
939
+
940
+ # Read a capsule's name (symbol key after normalization), as a string for
941
+ # comparison. Returns nil for unnamed (legacy) entries.
855
942
  def capsule_name(entry)
856
- raw = entry[:name] || entry["name"]
857
- raw&.to_s
943
+ entry[:name]&.to_s
858
944
  end
859
945
 
860
946
  # Auto-assign :name to parsed capsules where the first queue token would
@@ -887,7 +973,7 @@ module Pgbus
887
973
  existing_named = (@workers || []).select { |c| capsule_name(c) }
888
974
  return if existing_named.empty?
889
975
 
890
- existing_queues = existing_named.flat_map { |c| c[:queues] || c["queues"] || [] }
976
+ existing_queues = existing_named.flat_map { |c| c[:queues] || [] }
891
977
  return if existing_queues.empty?
892
978
 
893
979
  if existing_queues.include?(CapsuleDSL::WILDCARD)
@@ -915,7 +1001,7 @@ module Pgbus
915
1001
  return 0 unless entries
916
1002
 
917
1003
  entries.sum do |entry|
918
- threads = entry[:threads] || entry["threads"] || default_threads
1004
+ threads = entry[:threads] || default_threads
919
1005
  unless threads.is_a?(Integer) && threads.positive?
920
1006
  raise ArgumentError,
921
1007
  "#{group} threads must be a positive integer, got #{threads.inspect}"
@@ -60,11 +60,19 @@ module Pgbus
60
60
 
61
61
  def evict(key)
62
62
  @cache.delete(key)
63
+ # Concurrent::Array#delete is O(n), but this only runs on expiry and n is
64
+ # bounded by @max_size, so it stays cheap. Without it, @insertion_order
65
+ # leaks and stale duplicates can evict live keys in evict_oldest.
66
+ @insertion_order.delete(key)
63
67
  end
64
68
 
65
69
  def evict_oldest
66
70
  while @cache.size >= @max_size && !@insertion_order.empty?
67
71
  oldest_key = @insertion_order.shift
72
+ # Skip residual order entries for keys already gone from @cache so a
73
+ # stale duplicate never evicts a freshly re-marked (live) key.
74
+ next unless @cache.key?(oldest_key)
75
+
68
76
  @cache.delete(oldest_key)
69
77
  end
70
78
  end
@@ -0,0 +1,250 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "uri"
4
+ require "pgbus/mcp/health_analyzer"
5
+
6
+ module Pgbus
7
+ # Preflight diagnostics for a pgbus deployment — the single command that
8
+ # answers "is this environment healthy enough to run?". Runs six checks and
9
+ # returns a machine-readable result plus a human report, so `pgbus doctor`
10
+ # and `rake pgbus:doctor` can gate a deploy or CI run (exit 0 on success,
11
+ # 1 on any failure).
12
+ #
13
+ # Doctor never touches PGMQ or PostgreSQL directly: every probe goes through
14
+ # Pgbus::Client (DB, PGMQ schema, queues, NOTIFY) or Pgbus::Web::DataSource
15
+ # (process liveness, via Pgbus::MCP::HealthAnalyzer). It also never raises —
16
+ # a broken environment turns into :fail results, never a crash — so it is
17
+ # safe to run against a database that is down or a half-installed schema.
18
+ class Doctor
19
+ # A single check result. status is one of :ok, :warn, :fail.
20
+ Check = Struct.new(:name, :status, :detail) do
21
+ def to_h
22
+ { name: name, status: status, detail: detail }
23
+ end
24
+ end
25
+
26
+ STATUS_ICON = { ok: "✓", warn: "!", fail: "✗" }.freeze
27
+
28
+ def initialize(config: Pgbus.configuration, client: Pgbus.client, data_source: nil)
29
+ @config = config
30
+ @client = client
31
+ @data_source = data_source || Pgbus::Web::DataSource.new(client: client)
32
+ end
33
+
34
+ # Run all checks and return an array of result hashes:
35
+ # { name:, status: :ok|:warn|:fail, detail: }
36
+ def run
37
+ @run ||= [
38
+ check_configuration,
39
+ check_database,
40
+ check_pgmq_schema,
41
+ check_queues,
42
+ check_notify,
43
+ check_processes
44
+ ].map(&:to_h)
45
+ end
46
+
47
+ # True when no check failed. Warnings do not fail the run — they surface a
48
+ # concern (e.g. an outdated PGMQ schema) without blocking a deploy.
49
+ def success?
50
+ run.none? { |c| c[:status] == :fail }
51
+ end
52
+
53
+ # Human-readable report: one line per check, then a resolved-config summary
54
+ # with passwords redacted. Suitable for stdout in the CLI and rake task.
55
+ def report
56
+ lines = ["Pgbus Doctor", "=" * 40]
57
+ run.each do |check|
58
+ icon = STATUS_ICON.fetch(check[:status], "?")
59
+ lines << format("%<icon>s %<name>-22s %<detail>s", icon: icon, name: check[:name], detail: check[:detail])
60
+ end
61
+ lines << ""
62
+ lines << "Configuration"
63
+ lines << ("-" * 40)
64
+ config_summary.each { |key, value| lines << format(" %<key>-20s %<value>s", key: key, value: value) }
65
+ lines.join("\n")
66
+ end
67
+
68
+ # Resolved-config summary for the report. Password-bearing fields
69
+ # (database_url, connection_params) are redacted.
70
+ def config_summary
71
+ {
72
+ queue_prefix: @config.queue_prefix,
73
+ default_queue: @config.default_queue,
74
+ pgmq_schema_mode: @config.pgmq_schema_mode,
75
+ resolved_pool_size: safe { @config.resolved_pool_size },
76
+ listen_notify: @config.listen_notify,
77
+ roles: @config.roles || "all",
78
+ capsules: capsule_summary,
79
+ database_url: redacted_database_url,
80
+ connection_params: redacted_connection_params
81
+ }.compact
82
+ end
83
+
84
+ private
85
+
86
+ # 1. Configuration validity.
87
+ def check_configuration
88
+ @config.validate!
89
+ Check.new(name: "Configuration", status: :ok, detail: "valid")
90
+ rescue ArgumentError => e
91
+ Check.new(name: "Configuration", status: :fail, detail: e.message)
92
+ rescue StandardError => e
93
+ Check.new(name: "Configuration", status: :fail, detail: "#{e.class}: #{e.message}")
94
+ end
95
+
96
+ # 2. Database connectivity — SELECT 1 via Client#ping.
97
+ def check_database
98
+ @client.ping
99
+ Check.new(name: "Database", status: :ok, detail: "reachable")
100
+ rescue StandardError => e
101
+ Check.new(name: "Database", status: :fail, detail: "#{e.class}: #{e.message}")
102
+ end
103
+
104
+ # 3. PGMQ schema presence + installed-vs-vendored version.
105
+ #
106
+ # A nil version does NOT automatically mean "not installed": PGMQ installed
107
+ # via the extension, or before pgbus added version tracking, leaves the
108
+ # pgmq schema fully working with no row in pgbus_pgmq_schema_versions. So we
109
+ # distinguish, mirroring `pgbus:pgmq:status`: schema absent → fail; schema
110
+ # present but untracked → warn; tracked but behind → warn; current → ok.
111
+ def check_pgmq_schema
112
+ installed = @client.pgmq_schema_version
113
+ latest = Pgbus::PgmqSchema.latest_version
114
+
115
+ if installed.nil?
116
+ unless @client.pgmq_installed?
117
+ return Check.new(name: "PGMQ schema", status: :fail,
118
+ detail: "not installed — run `rails generate pgbus:install`")
119
+ end
120
+
121
+ return Check.new(name: "PGMQ schema", status: :warn,
122
+ detail: "installed but no version tracking — run `rails generate pgbus:upgrade_pgmq`")
123
+ end
124
+
125
+ if Gem::Version.new(installed) < Gem::Version.new(latest)
126
+ Check.new(name: "PGMQ schema", status: :warn,
127
+ detail: "installed #{installed}, vendored #{latest} — run `rails generate pgbus:upgrade_pgmq`")
128
+ else
129
+ Check.new(name: "PGMQ schema", status: :ok, detail: "up to date (#{installed})")
130
+ end
131
+ rescue StandardError => e
132
+ Check.new(name: "PGMQ schema", status: :fail, detail: "#{e.class}: #{e.message}")
133
+ end
134
+
135
+ # 4. Queue existence — every configured queue must have its PGMQ table(s).
136
+ # Resolve each logical queue to the PHYSICAL names bootstrap creates (via the
137
+ # client's queue strategy) so a priority queue's _p0.._pN sub-tables are what
138
+ # we diff against list_queues — not the bare prefixed name priority mode
139
+ # never creates.
140
+ def check_queues
141
+ configured = @client.configured_queues.flat_map { |q| @client.physical_queue_names(q) }
142
+ existing = existing_queue_names
143
+ missing = configured - existing
144
+
145
+ if missing.empty?
146
+ Check.new(name: "Queues", status: :ok, detail: "#{configured.size} configured queue(s) present")
147
+ else
148
+ Check.new(name: "Queues", status: :fail, detail: "missing queue(s): #{missing.join(", ")}")
149
+ end
150
+ rescue StandardError => e
151
+ Check.new(name: "Queues", status: :fail, detail: "#{e.class}: #{e.message}")
152
+ end
153
+
154
+ # 5. LISTEN/NOTIFY — when configured on, every configured queue should carry
155
+ # the insert-NOTIFY trigger. A missing trigger is a warning, not a failure:
156
+ # pgbus still works (it falls back to polling), it just loses instant wakeup.
157
+ def check_notify
158
+ return Check.new(name: "LISTEN/NOTIFY", status: :ok, detail: "disabled in config") unless @config.listen_notify
159
+
160
+ without_trigger = @client.configured_queues.reject { |q| @client.notify_enabled?(q) }
161
+ if without_trigger.empty?
162
+ Check.new(name: "LISTEN/NOTIFY", status: :ok, detail: "triggers live on all configured queues")
163
+ else
164
+ Check.new(name: "LISTEN/NOTIFY", status: :warn,
165
+ detail: "no insert trigger on: #{without_trigger.join(", ")} (falling back to polling)")
166
+ end
167
+ rescue StandardError => e
168
+ Check.new(name: "LISTEN/NOTIFY", status: :fail, detail: "#{e.class}: #{e.message}")
169
+ end
170
+
171
+ # 6. Process liveness — the top-level health verdict (OK/DEGRADED/STALLED)
172
+ # from the shared HealthAnalyzer. STALLED (the silent-worker-wedge) is a
173
+ # failure; DEGRADED is a warning; OK passes.
174
+ def check_processes
175
+ verdict = Pgbus::MCP::HealthAnalyzer.new(@data_source).verdict
176
+ status = verdict[:status]
177
+ detail = Array(verdict[:reasons]).first || status
178
+
179
+ case status
180
+ when "STALLED" then Check.new(name: "Process liveness", status: :fail, detail: detail)
181
+ when "DEGRADED" then Check.new(name: "Process liveness", status: :warn, detail: detail)
182
+ else Check.new(name: "Process liveness", status: :ok, detail: "OK")
183
+ end
184
+ rescue StandardError => e
185
+ Check.new(name: "Process liveness", status: :fail, detail: "#{e.class}: #{e.message}")
186
+ end
187
+
188
+ # Physical queue names known to PGMQ. list_queues rows may be Hashes (with a
189
+ # :queue_name key) or value objects responding to #queue_name.
190
+ def existing_queue_names
191
+ Array(@client.list_queues).map do |row|
192
+ if row.respond_to?(:queue_name)
193
+ row.queue_name
194
+ elsif row.is_a?(Hash)
195
+ row[:queue_name] || row["queue_name"]
196
+ end
197
+ end.compact
198
+ end
199
+
200
+ def capsule_summary
201
+ Array(@config.workers).map { |w| w[:name] || (w[:queues] || []).join("+") }.join(", ")
202
+ end
203
+
204
+ # Redact the password from a libpq URL/URI while keeping host/db visible so
205
+ # the operator can still confirm which database they pointed at.
206
+ def redacted_database_url
207
+ return nil unless @config.database_url
208
+
209
+ redact_url(@config.database_url)
210
+ end
211
+
212
+ # connection_params is a free-form libpq keyword hash, so redact every key
213
+ # whose name looks secret (password, sslpassword, ...) — not just :password —
214
+ # rather than assuming a fixed shape. Preserves symbol/string key form.
215
+ SECRET_KEY_PATTERN = /pass|secret|token/i
216
+ private_constant :SECRET_KEY_PATTERN
217
+
218
+ def redacted_connection_params
219
+ params = @config.connection_params
220
+ return nil unless params.is_a?(Hash)
221
+
222
+ params.each_with_object({}) do |(key, value), out|
223
+ out[key] = key.to_s.match?(SECRET_KEY_PATTERN) ? "[REDACTED]" : value
224
+ end
225
+ end
226
+
227
+ # Replace the password in a userinfo authority (scheme://user:pass@host) or a
228
+ # key=value conninfo string (password=secret). Falls back to a blanket
229
+ # redaction label if parsing fails, never leaking the original.
230
+ #
231
+ # The userinfo password can itself contain '@' and ':' (common in generated
232
+ # secrets), so match it greedily up to the LAST '@' before the authority's
233
+ # host — a lazy `[^@]+` would stop at the first '@' and leak the remainder.
234
+ def redact_url(url)
235
+ redacted = url.sub(%r{(://[^:/@]+:).+(@[^@/]*(?:[/?]|\z))}, '\1[REDACTED]\2')
236
+ redacted.gsub(/(\bpassword=)('[^']*'|[^\s'"]+)/i, '\1[REDACTED]')
237
+ rescue StandardError
238
+ "[REDACTED]"
239
+ end
240
+
241
+ # Wrap a value-producing block so a broken resolver (e.g. resolved_pool_size
242
+ # raising on a malformed worker config) degrades to nil instead of raising
243
+ # out of config_summary.
244
+ def safe
245
+ yield
246
+ rescue StandardError
247
+ nil
248
+ end
249
+ end
250
+ end
data/lib/pgbus/engine.rb CHANGED
@@ -60,6 +60,7 @@ module Pgbus
60
60
  load File.expand_path("../tasks/pgbus_pgmq.rake", __dir__)
61
61
  load File.expand_path("../tasks/pgbus_streams.rake", __dir__)
62
62
  load File.expand_path("../tasks/pgbus_autovacuum.rake", __dir__)
63
+ load File.expand_path("../tasks/pgbus_doctor.rake", __dir__)
63
64
  end
64
65
 
65
66
  initializer "pgbus.i18n" do
@@ -86,6 +87,20 @@ module Pgbus
86
87
  end
87
88
  end
88
89
 
90
+ # Generic metrics adapter. Opt-in via config.metrics_backend; nil (default)
91
+ # installs no subscription, so there is zero overhead when unused. Runs
92
+ # independently of AppSignal — both consume the same pgbus.* events and both
93
+ # can be active at once.
94
+ initializer "pgbus.metrics", after: :load_config_initializers do
95
+ ActiveSupport.on_load(:after_initialize) do
96
+ next if Pgbus.configuration.metrics_backend.nil?
97
+
98
+ backend = Pgbus::Metrics.resolve_backend(Pgbus.configuration.metrics_backend)
99
+ Pgbus::Metrics::Subscriber.install!(backend: backend)
100
+ Pgbus.logger.info { "[Pgbus] Metrics subscriber installed (#{backend.class})" }
101
+ end
102
+ end
103
+
89
104
  # Install the watermark cache middleware ahead of the app's own
90
105
  # middleware so the thread-local cache is cleared between every
91
106
  # Rack request. Without this, repeated page renders served by the
@@ -46,6 +46,13 @@ module Pgbus
46
46
  available_capacity.positive?
47
47
  end
48
48
 
49
+ # True only when every slot is free — all in-flight work has finished.
50
+ # idle? answers "can this pool accept work?"; quiesced? answers
51
+ # "has this pool drained?". The worker drain loop needs the latter.
52
+ def quiesced?
53
+ available_capacity >= @capacity
54
+ end
55
+
49
56
  def shutdown
50
57
  @state_mutex.synchronize do
51
58
  return false if @shutdown_flag
@@ -37,6 +37,13 @@ module Pgbus
37
37
  available_capacity.positive?
38
38
  end
39
39
 
40
+ # True only when every slot is free — all in-flight work has finished.
41
+ # idle? answers "can this pool accept work?"; quiesced? answers
42
+ # "has this pool drained?". The worker drain loop needs the latter.
43
+ def quiesced?
44
+ available_capacity >= @capacity
45
+ end
46
+
40
47
  def shutdown
41
48
  @pool.shutdown
42
49
  end
@@ -11,6 +11,7 @@ module Pgbus
11
11
  # pgbus.client.send_batch — batch enqueue
12
12
  # pgbus.client.read_batch — batch dequeue
13
13
  # pgbus.client.read_message — single message dequeue
14
+ # pgbus.client.pool — connection pool snapshot (size/available/pool_timeout); emitted per worker heartbeat
14
15
  # pgbus.executor.execute — full job execution (deserialize + perform + archive)
15
16
  # pgbus.job_completed — job archived successfully
16
17
  # pgbus.job_failed — job raised; carries :exception_object
@@ -51,8 +51,9 @@ module Pgbus
51
51
 
52
52
  # The actual probe object; AppSignal calls #call once per minute.
53
53
  class Runner
54
- def initialize(data_source: nil)
54
+ def initialize(data_source: nil, client: nil)
55
55
  @data_source = data_source
56
+ @client = client
56
57
  @hostname = Socket.gethostname
57
58
  end
58
59
 
@@ -63,6 +64,7 @@ module Pgbus
63
64
  track_processes
64
65
  track_summary
65
66
  track_streams
67
+ track_pool
66
68
  end
67
69
 
68
70
  private
@@ -72,6 +74,10 @@ module Pgbus
72
74
  (::Pgbus::Web::DataSource.new if defined?(::Pgbus::Web::DataSource))
73
75
  end
74
76
 
77
+ def client
78
+ @client ||= (::Pgbus.client if defined?(::Pgbus) && ::Pgbus.respond_to?(:client))
79
+ end
80
+
75
81
  def track_queues
76
82
  data_source.queues_with_metrics.each do |q|
77
83
  tags = { queue: q[:name] }
@@ -110,6 +116,22 @@ module Pgbus
110
116
  log_failure("summary metrics", e)
111
117
  end
112
118
 
119
+ # The PGMQ connection pool is per-process (each host owns its own
120
+ # pool), so — unlike the cluster-wide queue/summary gauges — these
121
+ # are tagged with the hostname, same policy as active_processes.
122
+ # pool_stats already rescues to {} internally; the outer rescue here
123
+ # keeps a probe iteration alive if the client itself is unavailable.
124
+ def track_pool
125
+ stats = client&.pool_stats || {}
126
+ return if stats.empty?
127
+
128
+ tags = { hostname: @hostname }
129
+ gauge "pool_size", stats[:size], tags
130
+ gauge "pool_available", stats[:available], tags
131
+ rescue StandardError => e
132
+ log_failure("pool metrics", e)
133
+ end
134
+
113
135
  def track_streams
114
136
  return unless data_source.respond_to?(:stream_stats_available?) &&
115
137
  data_source.stream_stats_available?