pgbus 0.9.7 → 0.9.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +367 -0
  3. data/README.md +454 -25
  4. data/Rakefile +10 -1
  5. data/app/helpers/pgbus/application_helper.rb +37 -0
  6. data/app/views/pgbus/processes/_processes_table.html.erb +4 -1
  7. data/config/locales/da.yml +4 -0
  8. data/config/locales/de.yml +4 -0
  9. data/config/locales/en.yml +4 -0
  10. data/config/locales/es.yml +4 -0
  11. data/config/locales/fi.yml +4 -0
  12. data/config/locales/fr.yml +4 -0
  13. data/config/locales/it.yml +4 -0
  14. data/config/locales/ja.yml +4 -0
  15. data/config/locales/nb.yml +4 -0
  16. data/config/locales/nl.yml +4 -0
  17. data/config/locales/pt.yml +4 -0
  18. data/config/locales/sv.yml +4 -0
  19. data/lib/generators/pgbus/{add_job_locks_generator.rb → add_uniqueness_keys_generator.rb} +5 -5
  20. data/lib/pgbus/active_job/adapter.rb +1 -1
  21. data/lib/pgbus/active_job/executor.rb +25 -4
  22. data/lib/pgbus/cli/dlq.rb +164 -0
  23. data/lib/pgbus/cli.rb +18 -1
  24. data/lib/pgbus/client/connection_health.rb +194 -0
  25. data/lib/pgbus/client.rb +592 -73
  26. data/lib/pgbus/config_loader.rb +50 -4
  27. data/lib/pgbus/configuration.rb +294 -79
  28. data/lib/pgbus/dedup_cache.rb +8 -0
  29. data/lib/pgbus/doctor.rb +275 -0
  30. data/lib/pgbus/engine.rb +15 -0
  31. data/lib/pgbus/execution_pools/async_pool.rb +9 -2
  32. data/lib/pgbus/execution_pools/thread_pool.rb +7 -0
  33. data/lib/pgbus/generators/config_converter.rb +7 -5
  34. data/lib/pgbus/generators/migration_detector.rb +20 -16
  35. data/lib/pgbus/instrumentation.rb +3 -1
  36. data/lib/pgbus/integrations/appsignal/probe.rb +23 -1
  37. data/lib/pgbus/integrations/appsignal/subscriber.rb +17 -2
  38. data/lib/pgbus/metrics/backend.rb +38 -0
  39. data/lib/pgbus/metrics/backends/prometheus.rb +123 -0
  40. data/lib/pgbus/metrics/backends/statsd.rb +64 -0
  41. data/lib/pgbus/metrics/prometheus_exporter.rb +34 -0
  42. data/lib/pgbus/metrics/subscriber.rb +214 -0
  43. data/lib/pgbus/metrics.rb +43 -0
  44. data/lib/pgbus/pgmq_schema/pgmq_v1.11.1.sql +2126 -0
  45. data/lib/pgbus/pgmq_schema.rb +7 -2
  46. data/lib/pgbus/process/consumer.rb +354 -18
  47. data/lib/pgbus/process/consumer_priority.rb +34 -0
  48. data/lib/pgbus/process/dispatcher.rb +265 -41
  49. data/lib/pgbus/process/heartbeat.rb +18 -5
  50. data/lib/pgbus/process/memory_usage.rb +48 -0
  51. data/lib/pgbus/process/notify_listener.rb +26 -7
  52. data/lib/pgbus/process/notify_probe.rb +96 -0
  53. data/lib/pgbus/process/primary_validator.rb +53 -0
  54. data/lib/pgbus/process/signal_handler.rb +6 -0
  55. data/lib/pgbus/process/supervisor.rb +423 -50
  56. data/lib/pgbus/process/worker.rb +288 -35
  57. data/lib/pgbus/recurring/schedule.rb +1 -2
  58. data/lib/pgbus/recurring/scheduler.rb +15 -1
  59. data/lib/pgbus/serializer.rb +4 -4
  60. data/lib/pgbus/streams/broadcastable_override.rb +0 -8
  61. data/lib/pgbus/streams/signed_name.rb +2 -2
  62. data/lib/pgbus/streams/turbo_broadcastable.rb +7 -5
  63. data/lib/pgbus/table_maintenance.rb +13 -2
  64. data/lib/pgbus/uniqueness.rb +11 -12
  65. data/lib/pgbus/version.rb +1 -1
  66. data/lib/pgbus/web/data_source.rb +36 -4
  67. data/lib/pgbus/web/health_app.rb +102 -0
  68. data/lib/pgbus/web/health_server.rb +144 -0
  69. data/lib/pgbus/web/payload_filter.rb +3 -3
  70. data/lib/pgbus/web/streamer/instance.rb +58 -2
  71. data/lib/pgbus/web/streamer/listener.rb +69 -21
  72. data/lib/pgbus.rb +77 -0
  73. data/lib/tasks/pgbus_doctor.rake +12 -0
  74. metadata +19 -4
  75. data/app/models/pgbus/job_lock.rb +0 -98
  76. data/lib/generators/pgbus/templates/add_job_locks.rb.erb +0 -21
@@ -1234,18 +1234,50 @@ module Pgbus
1234
1234
 
1235
1235
  def derive_process_status(stale, metadata, kind)
1236
1236
  return :stale if stale
1237
- return :healthy unless kind == "worker" && metadata.is_a?(Hash)
1237
+ return :healthy unless metadata.is_a?(Hash)
1238
+
1239
+ threshold = stall_threshold_for(kind)
1240
+ return :healthy unless threshold
1238
1241
 
1239
1242
  loop_tick = metadata["loop_tick_at"]
1240
1243
  return :healthy unless loop_tick
1241
1244
 
1242
- threshold = Pgbus.configuration.stall_threshold
1243
- return :healthy unless threshold&.positive?
1244
-
1245
1245
  age = Time.now.to_f - loop_tick.to_f
1246
1246
  age > threshold ? :stalled : :healthy
1247
1247
  end
1248
1248
 
1249
+ # Kind-aware stall threshold. Workers run a tight loop, so the base
1250
+ # stall_threshold applies directly. The dispatcher and scheduler sleep
1251
+ # between iterations, so their beacon is naturally up to one sleep
1252
+ # interval stale even when healthy — widen the window by that interval to
1253
+ # avoid false stalls. Consumers idle on an empty-read wait that can reach
1254
+ # NOTIFY_FALLBACK_POLL_SECONDS (15s) when a live listener drives wake-up,
1255
+ # so widen by that max wait when notify is on, or by the poll interval
1256
+ # otherwise. Kinds with no loop beacon (nil) are never stalled.
1257
+ def stall_threshold_for(kind)
1258
+ config = Pgbus.configuration
1259
+ base = config.stall_threshold
1260
+ return nil unless base&.positive?
1261
+
1262
+ case kind
1263
+ when "worker" then base
1264
+ when "dispatcher" then base + config.dispatch_interval
1265
+ when "scheduler" then base + config.recurring_schedule_interval
1266
+ when "consumer" then base + consumer_stall_slack(config)
1267
+ end
1268
+ end
1269
+
1270
+ # Widen the consumer's stall window by its maximum empty-read wait. With
1271
+ # notify wake-up live, that wait rises to NOTIFY_FALLBACK_POLL_SECONDS as a
1272
+ # safety net; without it, the consumer polls at polling_interval.
1273
+ def consumer_stall_slack(config)
1274
+ if config.worker_notify_wakeup?
1275
+ [config.polling_interval, Pgbus::Process::Consumer::NOTIFY_FALLBACK_POLL_SECONDS].max
1276
+ else
1277
+ config.polling_interval
1278
+ end
1279
+ end
1280
+
1249
1281
  def format_batch(record)
1250
1282
  total = record.total_jobs
1251
1283
  done = record.completed_jobs + record.discarded_jobs
@@ -0,0 +1,102 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module Pgbus
6
+ module Web
7
+ # Plain Rack app exposing HTTP liveness and readiness probes for
8
+ # orchestrators (Kubernetes kubelet, load balancers). Mount it in the host
9
+ # Rails app — no auth, no DB access on the liveness path:
10
+ #
11
+ # # config/routes.rb
12
+ # mount Pgbus::Web::HealthApp.new => "/pgbus/health"
13
+ #
14
+ # # kubelet
15
+ # livenessProbe: { httpGet: { path: /pgbus/health/livez, port: 3000 } }
16
+ # readinessProbe: { httpGet: { path: /pgbus/health/readyz, port: 3000 } }
17
+ #
18
+ # The supervisor process can also serve these two paths standalone via
19
+ # {Pgbus::Web::HealthServer} when `config.health_port` is set — that path
20
+ # synthesizes a minimal Rack env and calls this same #call, so the routing
21
+ # and verdict logic live in exactly one place.
22
+ #
23
+ # Routing (env-only, no Rack::Request so it works under the bare-socket
24
+ # HealthServer too):
25
+ # GET /livez → 200 text/plain "ok", unconditionally, NO database access
26
+ # (the serving process is up — that is all liveness means).
27
+ # GET /readyz → build a DataSource, run MCP::HealthAnalyzer#verdict:
28
+ # OK / DEGRADED → 200, STALLED → 503, body = verdict JSON.
29
+ # Any StandardError (DB unreachable) → 503 {"status":"ERROR"},
30
+ # logged via Pgbus.logger (never swallowed).
31
+ # unknown path → 404; non-GET on a known path → 405.
32
+ class HealthApp
33
+ LIVEZ = "/livez"
34
+ READYZ = "/readyz"
35
+ KNOWN_PATHS = [LIVEZ, READYZ].freeze
36
+ private_constant :KNOWN_PATHS
37
+
38
+ TEXT = { "Content-Type" => "text/plain" }.freeze
39
+ JSON_HEADERS = { "Content-Type" => "application/json" }.freeze
40
+ private_constant :TEXT, :JSON_HEADERS
41
+
42
+ # DEGRADED is deliberately "ready": the serving process can still answer,
43
+ # and flapping a pod out of rotation on a transient DEGRADED (a stale
44
+ # sibling, a paused queue) would remove capacity for no gain. Only STALLED
45
+ # — the silent-wedge signal — fails readiness.
46
+ READY_STATUSES = %w[OK DEGRADED].freeze
47
+ private_constant :READY_STATUSES
48
+
49
+ # @param data_source [Pgbus::Web::DataSource, nil] read layer for /readyz.
50
+ # nil (the default) builds a fresh DataSource per readiness check, which
51
+ # avoids serving stale metrics from a long-lived app's memoized instance.
52
+ def initialize(data_source: nil)
53
+ @data_source = data_source
54
+ end
55
+
56
+ def call(env)
57
+ path = env["PATH_INFO"].to_s
58
+ return not_found unless KNOWN_PATHS.include?(path)
59
+ return method_not_allowed unless env["REQUEST_METHOD"] == "GET"
60
+
61
+ path == LIVEZ ? livez : readyz
62
+ end
63
+
64
+ private
65
+
66
+ # Liveness: the process is running. No DB, no analyzer, no config lookups.
67
+ def livez
68
+ [200, TEXT.dup, ["ok"]]
69
+ end
70
+
71
+ def readyz
72
+ # HealthAnalyzer lives in the MCP namespace, which is excluded from
73
+ # Zeitwerk (its *tools* subclass the optional `mcp` gem). The analyzer
74
+ # itself has no gem dependency, so require just that one file — the
75
+ # standalone health server must work without the `mcp` gem installed.
76
+ require "pgbus/mcp/health_analyzer" unless defined?(Pgbus::MCP::HealthAnalyzer)
77
+
78
+ verdict = Pgbus::MCP::HealthAnalyzer.new(data_source_for_check).verdict
79
+ status = READY_STATUSES.include?(verdict[:status].to_s) ? 200 : 503
80
+ [status, JSON_HEADERS.dup, [verdict.to_json]]
81
+ rescue StandardError => e
82
+ Pgbus.logger.error { "[Pgbus::Web::HealthApp] readiness check failed: #{e.class}: #{e.message}" }
83
+ [503, JSON_HEADERS.dup, [{ status: "ERROR", error: e.message }.to_json]]
84
+ end
85
+
86
+ # Reuse an injected DataSource (tests, an app that wants one shared
87
+ # instance); otherwise build a fresh one each check so per-instance
88
+ # memoization can never serve stale queue/process metrics.
89
+ def data_source_for_check
90
+ @data_source || DataSource.new
91
+ end
92
+
93
+ def not_found
94
+ [404, TEXT.dup, ["pgbus: not found"]]
95
+ end
96
+
97
+ def method_not_allowed
98
+ [405, TEXT.merge("Allow" => "GET"), ["pgbus: method not allowed"]]
99
+ end
100
+ end
101
+ end
102
+ end
@@ -0,0 +1,144 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "socket"
4
+
5
+ module Pgbus
6
+ module Web
7
+ # Standalone HTTP health server for the supervisor process. When
8
+ # `config.health_port` is set, Supervisor#run starts one of these so a
9
+ # Kubernetes kubelet can probe the supervisor directly — the process that
10
+ # forks and watches workers — without the supervisor having to boot Rails,
11
+ # Puma, or the dashboard.
12
+ #
13
+ # It is intentionally tiny: one accept-loop thread over a TCPServer that
14
+ # reads only the HTTP request line (`GET /readyz HTTP/1.1`), synthesizes a
15
+ # minimal Rack env, and hands it to {HealthApp#call}. All routing and the
16
+ # OK/DEGRADED/STALLED → status mapping live in HealthApp, so the mounted-in-
17
+ # Rails path and this standalone path share exactly one implementation.
18
+ #
19
+ # This is a health probe surface, not a general HTTP server: it speaks just
20
+ # enough HTTP/1.0 to answer a kubelet. It only parses the request line
21
+ # (method + path), ignores headers and body, and closes the connection
22
+ # after each response (no keep-alive).
23
+ class HealthServer
24
+ # Cap the request line so a client can't stream unbounded data at the
25
+ # probe port. A real probe request line is well under 100 bytes.
26
+ MAX_REQUEST_LINE = 8_192
27
+ private_constant :MAX_REQUEST_LINE
28
+
29
+ STATUS_TEXT = {
30
+ 200 => "OK", 404 => "Not Found", 405 => "Method Not Allowed", 503 => "Service Unavailable"
31
+ }.freeze
32
+ private_constant :STATUS_TEXT
33
+
34
+ # @return [Integer, nil] the bound port. Equals the configured port, or
35
+ # the OS-assigned port when the configured port was 0 (used in tests).
36
+ # nil before #start.
37
+ attr_reader :port
38
+
39
+ def initialize(port:, bind: "127.0.0.1", app: nil, logger: nil)
40
+ @configured_port = port
41
+ @bind = bind
42
+ @app = app || HealthApp.new
43
+ @logger = logger
44
+ @server = nil
45
+ @thread = nil
46
+ @port = nil
47
+ end
48
+
49
+ # Bind the port and start the accept loop. Idempotent: a second call
50
+ # while running is a no-op (so a supervisor restart path can't double-bind).
51
+ def start
52
+ return if @server
53
+
54
+ @server = TCPServer.new(@bind, @configured_port)
55
+ @port = @server.addr[1]
56
+ @thread = Thread.new { accept_loop }
57
+ logger.info { "[Pgbus::Web::HealthServer] listening on #{@bind}:#{@port} (/livez, /readyz)" }
58
+ end
59
+
60
+ # Close the listening socket (unblocking the accept loop, which then
61
+ # exits) and join the thread. Safe to call when never started or twice.
62
+ def stop
63
+ server = @server
64
+ @server = nil
65
+ close_socket(server)
66
+ @thread&.join(2)
67
+ @thread = nil
68
+ @port = nil
69
+ end
70
+
71
+ private
72
+
73
+ # accept blocks until stop closes the socket, which raises here and ends
74
+ # the loop. Any per-connection error is logged and swallowed so one bad
75
+ # client can never take the probe surface down.
76
+ def accept_loop
77
+ loop do
78
+ server = @server
79
+ break unless server
80
+
81
+ client =
82
+ begin
83
+ server.accept
84
+ rescue IOError, Errno::EBADF, Errno::EINVAL
85
+ break # socket closed by #stop
86
+ end
87
+
88
+ handle_client(client)
89
+ end
90
+ end
91
+
92
+ def handle_client(client)
93
+ method, path = read_request_line(client)
94
+ return unless method
95
+
96
+ status, headers, body = @app.call(rack_env(method, path))
97
+ write_response(client, status, headers, body)
98
+ rescue StandardError => e
99
+ logger.warn { "[Pgbus::Web::HealthServer] connection error: #{e.class}: #{e.message}" }
100
+ ensure
101
+ close_socket(client)
102
+ end
103
+
104
+ # Read and parse only the first line: "METHOD PATH HTTP/x.y". Returns
105
+ # [method, path] or [nil, nil] when the line is missing/garbage so the
106
+ # caller drops the connection without dispatching.
107
+ def read_request_line(client)
108
+ line = client.gets("\r\n", MAX_REQUEST_LINE)
109
+ return [nil, nil] if line.nil?
110
+
111
+ method, path, = line.strip.split(" ", 3)
112
+ return [nil, nil] if method.nil? || path.nil?
113
+
114
+ [method, path]
115
+ end
116
+
117
+ # Minimal Rack env — HealthApp reads only REQUEST_METHOD and PATH_INFO.
118
+ # Strip any query string from the path so PATH_INFO stays a bare path.
119
+ def rack_env(method, path)
120
+ { "REQUEST_METHOD" => method, "PATH_INFO" => path.split("?", 2).first }
121
+ end
122
+
123
+ def write_response(client, status, headers, body)
124
+ reason = STATUS_TEXT.fetch(status, "OK")
125
+ payload = Array(body).join
126
+ lines = ["HTTP/1.0 #{status} #{reason}"]
127
+ headers.each { |k, v| lines << "#{k}: #{v}" }
128
+ lines << "Content-Length: #{payload.bytesize}"
129
+ lines << "Connection: close"
130
+ client.write("#{lines.join("\r\n")}\r\n\r\n#{payload}")
131
+ end
132
+
133
+ def close_socket(socket)
134
+ socket&.close
135
+ rescue IOError, Errno::EBADF
136
+ nil
137
+ end
138
+
139
+ def logger
140
+ @logger || Pgbus.logger
141
+ end
142
+ end
143
+ end
144
+ end
@@ -16,7 +16,7 @@ module Pgbus
16
16
  FILTERED = "[FILTERED]"
17
17
 
18
18
  def filter(value)
19
- return value unless Pgbus.configuration.dashboard_filter_sensitive
19
+ return value unless Pgbus.configuration.web_filter_sensitive
20
20
 
21
21
  case value
22
22
  when Hash
@@ -29,7 +29,7 @@ module Pgbus
29
29
  end
30
30
 
31
31
  def filter_json(value)
32
- return value unless Pgbus.configuration.dashboard_filter_sensitive
32
+ return value unless Pgbus.configuration.web_filter_sensitive
33
33
 
34
34
  case value
35
35
  when nil then nil
@@ -46,7 +46,7 @@ module Pgbus
46
46
  end
47
47
 
48
48
  def parameter_filter
49
- patterns = Pgbus.configuration.dashboard_filter_parameters ||
49
+ patterns = Pgbus.configuration.web_filter_parameters ||
50
50
  rails_filter_parameters ||
51
51
  DEFAULT_FILTER_PATTERNS
52
52
  ActiveSupport::ParameterFilter.new(patterns)
@@ -28,7 +28,8 @@ module Pgbus
28
28
  pg_connection: nil,
29
29
  logger: Pgbus.logger,
30
30
  registry: nil,
31
- dispatch_queue: nil
31
+ dispatch_queue: nil,
32
+ connection_factory: nil
32
33
  )
33
34
  @client = client
34
35
  @config = config
@@ -42,7 +43,14 @@ module Pgbus
42
43
  pg_connection: @pg_connection,
43
44
  dispatch_queue: @dispatch_queue,
44
45
  health_check_ms: @config.streams_listen_health_check_ms,
45
- logger: @logger
46
+ logger: @logger,
47
+ # On reconnect the Listener rebuilds its OWN connection via this
48
+ # factory (fresh connect re-resolves DNS, converges on the promoted
49
+ # primary after a failover) instead of resetting a possibly-dead
50
+ # socket. Always provided — even when an initial pg_connection: is
51
+ # injected, the reconnect path builds a fresh raw connection. A test
52
+ # can inject its own factory to avoid touching real configuration.
53
+ connection_factory: connection_factory || -> { build_raw_pg_connection }
46
54
  )
47
55
  @dispatcher = StreamEventDispatcher.new(
48
56
  client: @client,
@@ -143,7 +151,20 @@ module Pgbus
143
151
  end
144
152
  end
145
153
 
154
+ # Build the INITIAL LISTEN connection: raw connect, reject a replica
155
+ # fatally (a misconfiguration at boot), then run the one-shot delivery
156
+ # self-probe. The reconnect path uses build_raw_pg_connection directly
157
+ # (validation retries there, probe is skipped) — see the factory wired
158
+ # into Listener.new above.
146
159
  def build_pg_connection
160
+ probe(validate_primary(build_raw_pg_connection))
161
+ end
162
+
163
+ # Raw PG.connect with no validation or probe. This is what the Listener's
164
+ # connection_factory calls on every reconnect attempt: the reconnect loop
165
+ # runs its own PrimaryValidator (retrying on a replica) and skips the
166
+ # probe to stay cheap, mirroring Pgbus::Process::NotifyListener.
167
+ def build_raw_pg_connection
147
168
  require "pg" unless defined?(::PG::Connection)
148
169
  opts = @config.streams_connection_options
149
170
  case opts
@@ -167,6 +188,41 @@ module Pgbus
167
188
  "Cannot build streamer PG connection from #{opts.class}"
168
189
  end
169
190
  end
191
+
192
+ # Reject a connection that landed on a read-only replica before the
193
+ # streamer wires it into a Listener. After a failover, stale DNS can
194
+ # point this fresh PG.connect at the demoted master; NOTIFY fires only
195
+ # on the primary, so the streamer would sit deaf. Unlike the reconnect
196
+ # loop (which retries), this runs once at Instance construction (Puma
197
+ # worker boot), so a replica here is a fatal misconfiguration: re-raise
198
+ # as a ConfigurationError naming the direct-connection overrides.
199
+ def validate_primary(pg_connection)
200
+ Pgbus::Process::PrimaryValidator.validate_primary!(pg_connection)
201
+ rescue Pgbus::Process::ReplicaConnectionError => e
202
+ close_quietly(pg_connection)
203
+ raise Pgbus::ConfigurationError,
204
+ "Streamer LISTEN connection landed on a read-only replica " \
205
+ "(#{e.message}). NOTIFY fires only on the primary, so the " \
206
+ "streamer would never wake. Point the streamer at a DIRECT " \
207
+ "primary via streams_database_url (or streams_host / " \
208
+ "streams_port)."
209
+ end
210
+
211
+ def close_quietly(pg_connection)
212
+ pg_connection.close if pg_connection.respond_to?(:close)
213
+ rescue StandardError
214
+ nil
215
+ end
216
+
217
+ # One-shot LISTEN/NOTIFY delivery self-probe on the freshly built
218
+ # connection. A transaction-mode pooler or replica that silently breaks
219
+ # NOTIFY is surfaced with an actionable error (naming the streams_*
220
+ # overrides); the streamer still starts and degrades to slow SSE
221
+ # updates rather than crashing. Returns the connection unchanged.
222
+ def probe(pg_connection)
223
+ Pgbus::Process::NotifyProbe.probe_notify_delivery!(pg_connection, logger: @logger)
224
+ pg_connection
225
+ end
170
226
  end
171
227
  end
172
228
  end
@@ -19,10 +19,11 @@ module Pgbus
19
19
  #
20
20
  # Health check: `wait_for_notify(timeout)` returns nil on timeout. When
21
21
  # it does, the listener runs `SELECT 1` as a TCP keepalive. If that
22
- # raises, the connection is reset (`conn.reset`) and every channel in
23
- # `@listening_to` is re-LISTENed. This is the fix for design doc §11 #1
24
- # (silently dropped LISTEN connections from NAT / PG restart / network
25
- # blips).
22
+ # raises, the listener rebuilds a FRESH connection via connection_factory
23
+ # and re-LISTENs every channel in `@listening_to`. This is the fix for
24
+ # design doc §11 #1 (silently dropped LISTEN connections from NAT / PG
25
+ # restart / network blips) — a fresh connect also re-resolves DNS and
26
+ # converges on the promoted primary after a failover.
26
27
  #
27
28
  # NOTIFY channel naming (from pgmq_v1.11.0.sql:1634):
28
29
  # PG_NOTIFY('pgmq.' || TG_TABLE_NAME || '.' || TG_OP, NULL)
@@ -38,13 +39,24 @@ module Pgbus
38
39
  CHANNEL_PREFIX = "pgmq.q_"
39
40
  CHANNEL_SUFFIX = ".INSERT"
40
41
 
42
+ RECONNECT_BACKOFF_SECONDS = 0.5
43
+
41
44
  attr_reader :listening_to
42
45
 
43
- def initialize(pg_connection:, dispatch_queue:, health_check_ms:, logger: Pgbus.logger)
46
+ # @param connection_factory [#call] builds a FRESH PG connection on each
47
+ # reconnect attempt (the Instance passes -> { build_raw_pg_connection }).
48
+ # Required — the reconnect loop always rebuilds rather than resetting a
49
+ # possibly-dead socket, so every caller (production and tests) must pass
50
+ # one.
51
+ def initialize(pg_connection:, dispatch_queue:, health_check_ms:,
52
+ connection_factory:, logger: Pgbus.logger)
53
+ raise ArgumentError, "connection_factory is required" unless connection_factory.respond_to?(:call)
54
+
44
55
  @conn = pg_connection
45
56
  @dispatch_queue = dispatch_queue
46
57
  @health_check_ms = health_check_ms
47
58
  @logger = logger
59
+ @connection_factory = connection_factory
48
60
  @listening_to = Set.new
49
61
  @commands = Queue.new
50
62
  @running = false
@@ -197,28 +209,64 @@ module Pgbus
197
209
  @conn.exec("SELECT 1")
198
210
  end
199
211
 
212
+ # Retry reconnect until we succeed (fresh conn + every channel
213
+ # re-LISTENed) or #stop flips @running to false. Mirrors
214
+ # NotifyListener#reconnect!: a single failed attempt never returns with
215
+ # a dead connection while running, so ensure_listening acks can't
216
+ # succeed vacuously against a broken LISTEN socket and strand SSE
217
+ # clients.
200
218
  def reconnect!
201
- @conn.reset
202
- # Don't clear @listening_to until the new set is built. If a
203
- # mid-loop LISTEN raises, we keep the original set so the
204
- # next reconnect cycle still knows which channels need to
205
- # come back. The previous version cleared first and lost
206
- # any channels not yet retried on a transient error.
207
- to_relisten = @listening_to.to_a
208
- new_listening = Set.new
209
- to_relisten.each do |channel|
210
- @conn.exec(%(LISTEN "#{channel}"))
211
- new_listening.add(channel)
219
+ # Snapshot the canonical subscription set BEFORE tearing down the old
220
+ # connection. We rebuild @listening_to from this and only publish it
221
+ # once every channel has been re-LISTENed on the new conn, so a
222
+ # mid-loop LISTEN failure never loses channels.
223
+ channels = @listening_to.to_a
224
+ loop do
225
+ return unless @running
226
+
227
+ close_quietly(@conn)
228
+ @conn = nil
229
+ new_conn = nil
230
+ begin
231
+ new_conn = @connection_factory.call
232
+ # Reject a replica before re-LISTENing. A fresh connect re-resolves
233
+ # DNS, so backing off and retrying converges on the promoted primary
234
+ # once DNS catches up after a failover. NOTIFY fires only on the
235
+ # primary; re-LISTENing on a replica registers channels that never wake.
236
+ Pgbus::Process::PrimaryValidator.validate_primary!(new_conn)
237
+ channels.each { |channel| new_conn.exec(%(LISTEN "#{channel}")) }
238
+ rescue PG::Error, Pgbus::Process::ReplicaConnectionError, Pgbus::ConfigurationError => e
239
+ # The factory may raise a ConfigurationError (a bad
240
+ # streams_connection_options, e.g. a pooled AR connection) or return
241
+ # a live conn before a later LISTEN raised (or validate_primary!
242
+ # rejected a replica). Close any partial conn so repeated failures
243
+ # don't leak PG connections, then back off and retry rather than
244
+ # letting the exception kill the listener thread silently.
245
+ close_quietly(new_conn)
246
+ @logger.error { "[Pgbus::Streamer::Listener] reconnect failed: #{e.class}: #{e.message}" }
247
+ sleep RECONNECT_BACKOFF_SECONDS
248
+ next
249
+ end
250
+
251
+ @conn = new_conn
252
+ @listening_to = Set.new(channels)
253
+ return
212
254
  end
213
- @listening_to = new_listening
214
- rescue PG::Error => e
215
- @logger.error { "[Pgbus::Streamer::Listener] reconnect failed: #{e.class}: #{e.message}" }
216
- sleep 0.5
255
+ end
256
+
257
+ # Close a PG connection we're discarding (the old conn at the top of a
258
+ # reconnect cycle, or a half-built one whose LISTEN raised). Best-effort.
259
+ def close_quietly(conn)
260
+ conn&.close if conn.respond_to?(:close)
261
+ rescue StandardError
262
+ nil
217
263
  end
218
264
 
219
265
  def safe_unlisten_all
220
266
  @listening_to.each do |channel|
221
- @conn.exec(%(UNLISTEN "#{channel}"))
267
+ # @conn may be nil if #stop interrupted the reconnect loop between
268
+ # closing the old connection and publishing a new one.
269
+ @conn&.exec(%(UNLISTEN "#{channel}"))
222
270
  rescue PG::Error
223
271
  # connection may be dead; nothing we can do
224
272
  end
data/lib/pgbus.rb CHANGED
@@ -10,6 +10,24 @@ module Pgbus
10
10
  # this to be configurable.
11
11
  DEAD_LETTER_SUFFIX = "_dlq"
12
12
 
13
+ # Error-hierarchy policy (the 1.0 contract, issue #282):
14
+ #
15
+ # * OPERATIONAL errors — a pgbus subsystem failed to do its job at runtime
16
+ # (bad configuration, a pool that's shutting down, a batch enqueue that
17
+ # came back short, a rejected GlobalID, a missing PGMQ version) — descend
18
+ # from Pgbus::Error. `rescue Pgbus::Error` is meant to catch every one of
19
+ # them; that is the whole point of the hierarchy.
20
+ #
21
+ # * ARGUMENT-SHAPE errors — the caller handed a method a malformed argument
22
+ # (a stream name too long for the queue-name budget, an unparseable
23
+ # cursor, a capsule DSL line that doesn't parse) — stay ArgumentError
24
+ # subclasses. That's what ArgumentError means, and rescuing it is the
25
+ # caller's responsibility, not an operational concern. These live in
26
+ # their own files: Streams::StreamNameTooLong, Streams::Cursor::InvalidCursor,
27
+ # Configuration::CapsuleDSL::ParseError.
28
+ #
29
+ # When adding a new error class, decide which half it belongs to and parent
30
+ # it accordingly — do not default to StandardError.
13
31
  class Error < StandardError; end
14
32
  class ConfigurationError < Error; end
15
33
  class SerializationError < Error; end
@@ -20,6 +38,41 @@ module Pgbus
20
38
  class SchemaNotReady < Error; end
21
39
  class ReadTimeoutError < Error; end
22
40
 
41
+ # Raised by the ActiveJob adapter when a batch enqueue (perform_all_later)
42
+ # comes back with a msg_id count that doesn't match the number of jobs sent —
43
+ # a data-integrity signal that some jobs may not have been persisted.
44
+ class EnqueueError < Error; end
45
+
46
+ # Raised by the execution pools when work can't be accepted: the pool is
47
+ # shutting down, or it's momentarily at capacity. Consumer-reachable during
48
+ # job submission, so it descends from Pgbus::Error rather than RuntimeError.
49
+ class ExecutionPoolError < Error; end
50
+
51
+ # Raised by Client read paths when the in-memory connection-health circuit
52
+ # breaker (Client::ConnectionHealth) is open: the database has failed enough
53
+ # consecutive connection attempts that reads now fail fast without a pool
54
+ # checkout, sparing a dead database from the whole fleet re-polling and the
55
+ # error tracker from per-poll noise. Workers rescue this and idle until the
56
+ # breaker's backoff window admits a probe. See issue #197.
57
+ class ConnectionCircuitOpenError < Error; end
58
+
59
+ module Process
60
+ # A LISTEN connection landed on a read-only replica (pg_is_in_recovery() =>
61
+ # t). After a failover, stale DNS can point a fresh connection at the
62
+ # demoted master; NOTIFY fires only on the primary, so such a connection
63
+ # connects "successfully" but never wakes. Raised by
64
+ # Process::PrimaryValidator so the listener rejects it and retries with
65
+ # backoff (re-resolving DNS each attempt).
66
+ #
67
+ # Defined here rather than in process/primary_validator.rb because Zeitwerk
68
+ # requires each managed file to define exactly the constant matching its
69
+ # path — a second class in that file would break eager_load. lib/pgbus.rb
70
+ # is the (require-loaded) gem entry point, so it can define the error and
71
+ # explicitly namespace Process without conflicting with Zeitwerk's
72
+ # management of the lib/pgbus/process/ directory.
73
+ class ReplicaConnectionError < Error; end
74
+ end
75
+
23
76
  class << self
24
77
  # Process-global flag set by Worker#graceful_shutdown so the adapter
25
78
  # can report stopping? to ActiveJob::Continuation (Rails 8.1+).
@@ -35,6 +88,7 @@ module Pgbus
35
88
  loader.inflector.inflect(
36
89
  "pgbus" => "Pgbus",
37
90
  "cli" => "CLI",
91
+ "dlq" => "DLQ",
38
92
  "dsl" => "DSL",
39
93
  "capsule_dsl" => "CapsuleDSL",
40
94
  "mcp" => "MCP"
@@ -92,6 +146,11 @@ module Pgbus
92
146
 
93
147
  def configure
94
148
  yield configuration
149
+ # Fail loud at boot on an invalid value rather than leaving it dormant
150
+ # until a worker path consumes it. Checking the flag after the yield
151
+ # lets a block opt out for itself via `c.eager_validation = false`.
152
+ configuration.validate! if configuration.eager_validation
153
+ configuration
95
154
  end
96
155
 
97
156
  def client
@@ -150,6 +209,24 @@ module Pgbus
150
209
  Streams::Key.stream_key!(key)
151
210
  end
152
211
 
212
+ # Publish an event to the bus — the top-level shortcut for
213
+ # `Pgbus::EventBus::Publisher.publish`, symmetric with `Pgbus.stream`.
214
+ #
215
+ # Pgbus.publish("orders.created", { order_id: 42 })
216
+ # Pgbus.publish("orders.created", order, headers: { "x-trace" => id }, delay: 30)
217
+ def publish(routing_key, payload, headers: nil, delay: 0)
218
+ EventBus::Publisher.publish(routing_key, payload, headers: headers, delay: delay)
219
+ end
220
+
221
+ # Publish an event with a delay. Mirrors
222
+ # `Pgbus::EventBus::Publisher.publish_later`; `delay:` is required (a
223
+ # publish_later with no delay is just publish).
224
+ #
225
+ # Pgbus.publish_later("orders.reminder", { order_id: 42 }, delay: 1.hour)
226
+ def publish_later(routing_key, payload, delay:, headers: nil)
227
+ EventBus::Publisher.publish_later(routing_key, payload, delay: delay, headers: headers)
228
+ end
229
+
153
230
  def reset!
154
231
  @client&.close
155
232
  @client = nil
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ namespace :pgbus do
4
+ desc "Run environment diagnostics (config, DB, PGMQ, queues, LISTEN/NOTIFY, process liveness)"
5
+ task doctor: :environment do
6
+ require "pgbus/doctor"
7
+
8
+ doctor = Pgbus::Doctor.new
9
+ puts doctor.report
10
+ exit 1 unless doctor.success?
11
+ end
12
+ end