pgbus 0.9.7 → 0.9.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +38 -0
  3. data/README.md +119 -1
  4. data/Rakefile +10 -1
  5. data/app/helpers/pgbus/application_helper.rb +37 -0
  6. data/app/views/pgbus/processes/_processes_table.html.erb +4 -1
  7. data/config/locales/da.yml +4 -0
  8. data/config/locales/de.yml +4 -0
  9. data/config/locales/en.yml +4 -0
  10. data/config/locales/es.yml +4 -0
  11. data/config/locales/fi.yml +4 -0
  12. data/config/locales/fr.yml +4 -0
  13. data/config/locales/it.yml +4 -0
  14. data/config/locales/ja.yml +4 -0
  15. data/config/locales/nb.yml +4 -0
  16. data/config/locales/nl.yml +4 -0
  17. data/config/locales/pt.yml +4 -0
  18. data/config/locales/sv.yml +4 -0
  19. data/lib/pgbus/active_job/executor.rb +25 -4
  20. data/lib/pgbus/cli/dlq.rb +164 -0
  21. data/lib/pgbus/cli.rb +18 -1
  22. data/lib/pgbus/client/connection_health.rb +194 -0
  23. data/lib/pgbus/client.rb +592 -73
  24. data/lib/pgbus/config_loader.rb +23 -4
  25. data/lib/pgbus/configuration.rb +98 -12
  26. data/lib/pgbus/dedup_cache.rb +8 -0
  27. data/lib/pgbus/doctor.rb +250 -0
  28. data/lib/pgbus/engine.rb +15 -0
  29. data/lib/pgbus/execution_pools/async_pool.rb +7 -0
  30. data/lib/pgbus/execution_pools/thread_pool.rb +7 -0
  31. data/lib/pgbus/instrumentation.rb +1 -0
  32. data/lib/pgbus/integrations/appsignal/probe.rb +23 -1
  33. data/lib/pgbus/metrics/backend.rb +38 -0
  34. data/lib/pgbus/metrics/backends/prometheus.rb +123 -0
  35. data/lib/pgbus/metrics/backends/statsd.rb +64 -0
  36. data/lib/pgbus/metrics/prometheus_exporter.rb +34 -0
  37. data/lib/pgbus/metrics/subscriber.rb +190 -0
  38. data/lib/pgbus/metrics.rb +42 -0
  39. data/lib/pgbus/process/consumer.rb +215 -8
  40. data/lib/pgbus/process/consumer_priority.rb +34 -0
  41. data/lib/pgbus/process/dispatcher.rb +265 -41
  42. data/lib/pgbus/process/heartbeat.rb +18 -5
  43. data/lib/pgbus/process/memory_usage.rb +48 -0
  44. data/lib/pgbus/process/notify_listener.rb +26 -7
  45. data/lib/pgbus/process/notify_probe.rb +96 -0
  46. data/lib/pgbus/process/primary_validator.rb +53 -0
  47. data/lib/pgbus/process/signal_handler.rb +6 -0
  48. data/lib/pgbus/process/supervisor.rb +396 -46
  49. data/lib/pgbus/process/worker.rb +298 -35
  50. data/lib/pgbus/recurring/scheduler.rb +15 -1
  51. data/lib/pgbus/streams/turbo_broadcastable.rb +7 -5
  52. data/lib/pgbus/table_maintenance.rb +13 -2
  53. data/lib/pgbus/version.rb +1 -1
  54. data/lib/pgbus/web/data_source.rb +20 -4
  55. data/lib/pgbus/web/health_app.rb +102 -0
  56. data/lib/pgbus/web/health_server.rb +144 -0
  57. data/lib/pgbus/web/streamer/instance.rb +55 -1
  58. data/lib/pgbus/web/streamer/listener.rb +72 -9
  59. data/lib/pgbus.rb +37 -0
  60. data/lib/rubocop/cop/pgbus/no_ruby_timeout.rb +42 -0
  61. data/lib/rubocop/pgbus.rb +5 -0
  62. data/lib/tasks/pgbus_doctor.rake +12 -0
  63. metadata +18 -1
@@ -0,0 +1,102 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module Pgbus
6
+ module Web
7
+ # Plain Rack app exposing HTTP liveness and readiness probes for
8
+ # orchestrators (Kubernetes kubelet, load balancers). Mount it in the host
9
+ # Rails app — no auth, no DB access on the liveness path:
10
+ #
11
+ # # config/routes.rb
12
+ # mount Pgbus::Web::HealthApp.new => "/pgbus/health"
13
+ #
14
+ # # kubelet
15
+ # livenessProbe: { httpGet: { path: /pgbus/health/livez, port: 3000 } }
16
+ # readinessProbe: { httpGet: { path: /pgbus/health/readyz, port: 3000 } }
17
+ #
18
+ # The supervisor process can also serve these two paths standalone via
19
+ # {Pgbus::Web::HealthServer} when `config.health_port` is set — that path
20
+ # synthesizes a minimal Rack env and calls this same #call, so the routing
21
+ # and verdict logic live in exactly one place.
22
+ #
23
+ # Routing (env-only, no Rack::Request so it works under the bare-socket
24
+ # HealthServer too):
25
+ # GET /livez → 200 text/plain "ok", unconditionally, NO database access
26
+ # (the serving process is up — that is all liveness means).
27
+ # GET /readyz → build a DataSource, run MCP::HealthAnalyzer#verdict:
28
+ # OK / DEGRADED → 200, STALLED → 503, body = verdict JSON.
29
+ # Any StandardError (DB unreachable) → 503 {"status":"ERROR"},
30
+ # logged via Pgbus.logger (never swallowed).
31
+ # unknown path → 404; non-GET on a known path → 405.
32
+ class HealthApp
33
+ LIVEZ = "/livez"
34
+ READYZ = "/readyz"
35
+ KNOWN_PATHS = [LIVEZ, READYZ].freeze
36
+ private_constant :KNOWN_PATHS
37
+
38
+ TEXT = { "Content-Type" => "text/plain" }.freeze
39
+ JSON_HEADERS = { "Content-Type" => "application/json" }.freeze
40
+ private_constant :TEXT, :JSON_HEADERS
41
+
42
+ # DEGRADED is deliberately "ready": the serving process can still answer,
43
+ # and flapping a pod out of rotation on a transient DEGRADED (a stale
44
+ # sibling, a paused queue) would remove capacity for no gain. Only STALLED
45
+ # — the silent-wedge signal — fails readiness.
46
+ READY_STATUSES = %w[OK DEGRADED].freeze
47
+ private_constant :READY_STATUSES
48
+
49
+ # @param data_source [Pgbus::Web::DataSource, nil] read layer for /readyz.
50
+ # nil (the default) builds a fresh DataSource per readiness check, which
51
+ # avoids serving stale metrics from a long-lived app's memoized instance.
52
+ def initialize(data_source: nil)
53
+ @data_source = data_source
54
+ end
55
+
56
+ def call(env)
57
+ path = env["PATH_INFO"].to_s
58
+ return not_found unless KNOWN_PATHS.include?(path)
59
+ return method_not_allowed unless env["REQUEST_METHOD"] == "GET"
60
+
61
+ path == LIVEZ ? livez : readyz
62
+ end
63
+
64
+ private
65
+
66
+ # Liveness: the process is running. No DB, no analyzer, no config lookups.
67
+ def livez
68
+ [200, TEXT.dup, ["ok"]]
69
+ end
70
+
71
+ def readyz
72
+ # HealthAnalyzer lives in the MCP namespace, which is excluded from
73
+ # Zeitwerk (its *tools* subclass the optional `mcp` gem). The analyzer
74
+ # itself has no gem dependency, so require just that one file — the
75
+ # standalone health server must work without the `mcp` gem installed.
76
+ require "pgbus/mcp/health_analyzer" unless defined?(Pgbus::MCP::HealthAnalyzer)
77
+
78
+ verdict = Pgbus::MCP::HealthAnalyzer.new(data_source_for_check).verdict
79
+ status = READY_STATUSES.include?(verdict[:status].to_s) ? 200 : 503
80
+ [status, JSON_HEADERS.dup, [verdict.to_json]]
81
+ rescue StandardError => e
82
+ Pgbus.logger.error { "[Pgbus::Web::HealthApp] readiness check failed: #{e.class}: #{e.message}" }
83
+ [503, JSON_HEADERS.dup, [{ status: "ERROR", error: e.message }.to_json]]
84
+ end
85
+
86
+ # Reuse an injected DataSource (tests, an app that wants one shared
87
+ # instance); otherwise build a fresh one each check so per-instance
88
+ # memoization can never serve stale queue/process metrics.
89
+ def data_source_for_check
90
+ @data_source || DataSource.new
91
+ end
92
+
93
+ def not_found
94
+ [404, TEXT.dup, ["pgbus: not found"]]
95
+ end
96
+
97
+ def method_not_allowed
98
+ [405, TEXT.merge("Allow" => "GET"), ["pgbus: method not allowed"]]
99
+ end
100
+ end
101
+ end
102
+ end
@@ -0,0 +1,144 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "socket"
4
+
5
+ module Pgbus
6
+ module Web
7
+ # Standalone HTTP health server for the supervisor process. When
8
+ # `config.health_port` is set, Supervisor#run starts one of these so a
9
+ # Kubernetes kubelet can probe the supervisor directly — the process that
10
+ # forks and watches workers — without the supervisor having to boot Rails,
11
+ # Puma, or the dashboard.
12
+ #
13
+ # It is intentionally tiny: one accept-loop thread over a TCPServer that
14
+ # reads only the HTTP request line (`GET /readyz HTTP/1.1`), synthesizes a
15
+ # minimal Rack env, and hands it to {HealthApp#call}. All routing and the
16
+ # OK/DEGRADED/STALLED → status mapping live in HealthApp, so the mounted-in-
17
+ # Rails path and this standalone path share exactly one implementation.
18
+ #
19
+ # This is a health probe surface, not a general HTTP server: it speaks just
20
+ # enough HTTP/1.0 to answer a kubelet. It only parses the request line
21
+ # (method + path), ignores headers and body, and closes the connection
22
+ # after each response (no keep-alive).
23
+ class HealthServer
24
+ # Cap the request line so a client can't stream unbounded data at the
25
+ # probe port. A real probe request line is well under 100 bytes.
26
+ MAX_REQUEST_LINE = 8_192
27
+ private_constant :MAX_REQUEST_LINE
28
+
29
+ STATUS_TEXT = {
30
+ 200 => "OK", 404 => "Not Found", 405 => "Method Not Allowed", 503 => "Service Unavailable"
31
+ }.freeze
32
+ private_constant :STATUS_TEXT
33
+
34
+ # @return [Integer, nil] the bound port. Equals the configured port, or
35
+ # the OS-assigned port when the configured port was 0 (used in tests).
36
+ # nil before #start.
37
+ attr_reader :port
38
+
39
+ def initialize(port:, bind: "127.0.0.1", app: nil, logger: nil)
40
+ @configured_port = port
41
+ @bind = bind
42
+ @app = app || HealthApp.new
43
+ @logger = logger
44
+ @server = nil
45
+ @thread = nil
46
+ @port = nil
47
+ end
48
+
49
+ # Bind the port and start the accept loop. Idempotent: a second call
50
+ # while running is a no-op (so a supervisor restart path can't double-bind).
51
+ def start
52
+ return if @server
53
+
54
+ @server = TCPServer.new(@bind, @configured_port)
55
+ @port = @server.addr[1]
56
+ @thread = Thread.new { accept_loop }
57
+ logger.info { "[Pgbus::Web::HealthServer] listening on #{@bind}:#{@port} (/livez, /readyz)" }
58
+ end
59
+
60
+ # Close the listening socket (unblocking the accept loop, which then
61
+ # exits) and join the thread. Safe to call when never started or twice.
62
+ def stop
63
+ server = @server
64
+ @server = nil
65
+ close_socket(server)
66
+ @thread&.join(2)
67
+ @thread = nil
68
+ @port = nil
69
+ end
70
+
71
+ private
72
+
73
+ # accept blocks until stop closes the socket, which raises here and ends
74
+ # the loop. Any per-connection error is logged and swallowed so one bad
75
+ # client can never take the probe surface down.
76
+ def accept_loop
77
+ loop do
78
+ server = @server
79
+ break unless server
80
+
81
+ client =
82
+ begin
83
+ server.accept
84
+ rescue IOError, Errno::EBADF, Errno::EINVAL
85
+ break # socket closed by #stop
86
+ end
87
+
88
+ handle_client(client)
89
+ end
90
+ end
91
+
92
+ def handle_client(client)
93
+ method, path = read_request_line(client)
94
+ return unless method
95
+
96
+ status, headers, body = @app.call(rack_env(method, path))
97
+ write_response(client, status, headers, body)
98
+ rescue StandardError => e
99
+ logger.warn { "[Pgbus::Web::HealthServer] connection error: #{e.class}: #{e.message}" }
100
+ ensure
101
+ close_socket(client)
102
+ end
103
+
104
+ # Read and parse only the first line: "METHOD PATH HTTP/x.y". Returns
105
+ # [method, path] or [nil, nil] when the line is missing/garbage so the
106
+ # caller drops the connection without dispatching.
107
+ def read_request_line(client)
108
+ line = client.gets("\r\n", MAX_REQUEST_LINE)
109
+ return [nil, nil] if line.nil?
110
+
111
+ method, path, = line.strip.split(" ", 3)
112
+ return [nil, nil] if method.nil? || path.nil?
113
+
114
+ [method, path]
115
+ end
116
+
117
+ # Minimal Rack env — HealthApp reads only REQUEST_METHOD and PATH_INFO.
118
+ # Strip any query string from the path so PATH_INFO stays a bare path.
119
+ def rack_env(method, path)
120
+ { "REQUEST_METHOD" => method, "PATH_INFO" => path.split("?", 2).first }
121
+ end
122
+
123
+ def write_response(client, status, headers, body)
124
+ reason = STATUS_TEXT.fetch(status, "OK")
125
+ payload = Array(body).join
126
+ lines = ["HTTP/1.0 #{status} #{reason}"]
127
+ headers.each { |k, v| lines << "#{k}: #{v}" }
128
+ lines << "Content-Length: #{payload.bytesize}"
129
+ lines << "Connection: close"
130
+ client.write("#{lines.join("\r\n")}\r\n\r\n#{payload}")
131
+ end
132
+
133
+ def close_socket(socket)
134
+ socket&.close
135
+ rescue IOError, Errno::EBADF
136
+ nil
137
+ end
138
+
139
+ def logger
140
+ @logger || Pgbus.logger
141
+ end
142
+ end
143
+ end
144
+ end
@@ -42,7 +42,13 @@ module Pgbus
42
42
  pg_connection: @pg_connection,
43
43
  dispatch_queue: @dispatch_queue,
44
44
  health_check_ms: @config.streams_listen_health_check_ms,
45
- logger: @logger
45
+ logger: @logger,
46
+ # On reconnect the Listener rebuilds its OWN connection via this
47
+ # factory (fresh connect re-resolves DNS, converges on the promoted
48
+ # primary after a failover) instead of resetting a possibly-dead
49
+ # socket. Skipped for an injected pg_connection: (unit tests) — a
50
+ # nil factory falls back to conn.reset.
51
+ connection_factory: pg_connection ? nil : -> { build_raw_pg_connection }
46
52
  )
47
53
  @dispatcher = StreamEventDispatcher.new(
48
54
  client: @client,
@@ -143,7 +149,20 @@ module Pgbus
143
149
  end
144
150
  end
145
151
 
152
+ # Build the INITIAL LISTEN connection: raw connect, reject a replica
153
+ # fatally (a misconfiguration at boot), then run the one-shot delivery
154
+ # self-probe. The reconnect path uses build_raw_pg_connection directly
155
+ # (validation retries there, probe is skipped) — see the factory wired
156
+ # into Listener.new above.
146
157
  def build_pg_connection
158
+ probe(validate_primary(build_raw_pg_connection))
159
+ end
160
+
161
+ # Raw PG.connect with no validation or probe. This is what the Listener's
162
+ # connection_factory calls on every reconnect attempt: the reconnect loop
163
+ # runs its own PrimaryValidator (retrying on a replica) and skips the
164
+ # probe to stay cheap, mirroring Pgbus::Process::NotifyListener.
165
+ def build_raw_pg_connection
147
166
  require "pg" unless defined?(::PG::Connection)
148
167
  opts = @config.streams_connection_options
149
168
  case opts
@@ -167,6 +186,41 @@ module Pgbus
167
186
  "Cannot build streamer PG connection from #{opts.class}"
168
187
  end
169
188
  end
189
+
190
+ # Reject a connection that landed on a read-only replica before the
191
+ # streamer wires it into a Listener. After a failover, stale DNS can
192
+ # point this fresh PG.connect at the demoted master; NOTIFY fires only
193
+ # on the primary, so the streamer would sit deaf. Unlike the reconnect
194
+ # loop (which retries), this runs once at Instance construction (Puma
195
+ # worker boot), so a replica here is a fatal misconfiguration: re-raise
196
+ # as a ConfigurationError naming the direct-connection overrides.
197
+ def validate_primary(pg_connection)
198
+ Pgbus::Process::PrimaryValidator.validate_primary!(pg_connection)
199
+ rescue Pgbus::Process::ReplicaConnectionError => e
200
+ close_quietly(pg_connection)
201
+ raise Pgbus::ConfigurationError,
202
+ "Streamer LISTEN connection landed on a read-only replica " \
203
+ "(#{e.message}). NOTIFY fires only on the primary, so the " \
204
+ "streamer would never wake. Point the streamer at a DIRECT " \
205
+ "primary via streams_database_url (or streams_host / " \
206
+ "streams_port)."
207
+ end
208
+
209
+ def close_quietly(pg_connection)
210
+ pg_connection.close if pg_connection.respond_to?(:close)
211
+ rescue StandardError
212
+ nil
213
+ end
214
+
215
+ # One-shot LISTEN/NOTIFY delivery self-probe on the freshly built
216
+ # connection. A transaction-mode pooler or replica that silently breaks
217
+ # NOTIFY is surfaced with an actionable error (naming the streams_*
218
+ # overrides); the streamer still starts and degrades to slow SSE
219
+ # updates rather than crashing. Returns the connection unchanged.
220
+ def probe(pg_connection)
221
+ Pgbus::Process::NotifyProbe.probe_notify_delivery!(pg_connection, logger: @logger)
222
+ pg_connection
223
+ end
170
224
  end
171
225
  end
172
226
  end
@@ -38,13 +38,21 @@ module Pgbus
38
38
  CHANNEL_PREFIX = "pgmq.q_"
39
39
  CHANNEL_SUFFIX = ".INSERT"
40
40
 
41
+ RECONNECT_BACKOFF_SECONDS = 0.5
42
+
41
43
  attr_reader :listening_to
42
44
 
43
- def initialize(pg_connection:, dispatch_queue:, health_check_ms:, logger: Pgbus.logger)
45
+ # @param connection_factory [#call, nil] builds a FRESH PG connection on
46
+ # each reconnect attempt (the Instance passes -> { build_pg_connection }).
47
+ # When nil (tests injecting only pg_connection:), reconnect falls back to
48
+ # the legacy single-shot @conn.reset.
49
+ def initialize(pg_connection:, dispatch_queue:, health_check_ms:,
50
+ logger: Pgbus.logger, connection_factory: nil)
44
51
  @conn = pg_connection
45
52
  @dispatch_queue = dispatch_queue
46
53
  @health_check_ms = health_check_ms
47
54
  @logger = logger
55
+ @connection_factory = connection_factory
48
56
  @listening_to = Set.new
49
57
  @commands = Queue.new
50
58
  @running = false
@@ -197,13 +205,58 @@ module Pgbus
197
205
  @conn.exec("SELECT 1")
198
206
  end
199
207
 
208
+ # Retry reconnect until we succeed (fresh conn + every channel
209
+ # re-LISTENed) or #stop flips @running to false. Mirrors
210
+ # NotifyListener#reconnect!: a single failed attempt never returns with
211
+ # a dead connection while running, so ensure_listening acks can't
212
+ # succeed vacuously against a broken LISTEN socket and strand SSE
213
+ # clients. When no connection_factory was injected (tests passing only
214
+ # pg_connection:), fall back to the legacy single-shot conn.reset.
200
215
  def reconnect!
216
+ return reconnect_via_reset unless @connection_factory
217
+
218
+ # Snapshot the canonical subscription set BEFORE tearing down the old
219
+ # connection. We rebuild @listening_to from this and only publish it
220
+ # once every channel has been re-LISTENed on the new conn, so a
221
+ # mid-loop LISTEN failure never loses channels.
222
+ channels = @listening_to.to_a
223
+ loop do
224
+ return unless @running
225
+
226
+ close_quietly(@conn)
227
+ @conn = nil
228
+ new_conn = nil
229
+ begin
230
+ new_conn = @connection_factory.call
231
+ # Reject a replica before re-LISTENing. A fresh connect re-resolves
232
+ # DNS, so backing off and retrying converges on the promoted primary
233
+ # once DNS catches up after a failover. NOTIFY fires only on the
234
+ # primary; re-LISTENing on a replica registers channels that never wake.
235
+ Pgbus::Process::PrimaryValidator.validate_primary!(new_conn)
236
+ channels.each { |channel| new_conn.exec(%(LISTEN "#{channel}")) }
237
+ rescue PG::Error, Pgbus::Process::ReplicaConnectionError => e
238
+ # The factory may have returned a live conn before a later LISTEN
239
+ # raised (or validate_primary! rejected a replica). Close the
240
+ # partial conn so repeated failures don't leak PG connections.
241
+ close_quietly(new_conn)
242
+ @logger.error { "[Pgbus::Streamer::Listener] reconnect failed: #{e.class}: #{e.message}" }
243
+ sleep RECONNECT_BACKOFF_SECONDS
244
+ next
245
+ end
246
+
247
+ @conn = new_conn
248
+ @listening_to = Set.new(channels)
249
+ return
250
+ end
251
+ end
252
+
253
+ # Legacy fallback for tests that inject only pg_connection: (no factory).
254
+ # A single conn.reset attempt; on failure it logs and backs off, leaving
255
+ # recovery to the next run_loop cycle. Kept for backwards compatibility
256
+ # with test wiring; production always injects a connection_factory.
257
+ def reconnect_via_reset
201
258
  @conn.reset
202
- # Don't clear @listening_to until the new set is built. If a
203
- # mid-loop LISTEN raises, we keep the original set so the
204
- # next reconnect cycle still knows which channels need to
205
- # come back. The previous version cleared first and lost
206
- # any channels not yet retried on a transient error.
259
+ Pgbus::Process::PrimaryValidator.validate_primary!(@conn)
207
260
  to_relisten = @listening_to.to_a
208
261
  new_listening = Set.new
209
262
  to_relisten.each do |channel|
@@ -211,14 +264,24 @@ module Pgbus
211
264
  new_listening.add(channel)
212
265
  end
213
266
  @listening_to = new_listening
214
- rescue PG::Error => e
267
+ rescue PG::Error, Pgbus::Process::ReplicaConnectionError => e
215
268
  @logger.error { "[Pgbus::Streamer::Listener] reconnect failed: #{e.class}: #{e.message}" }
216
- sleep 0.5
269
+ sleep RECONNECT_BACKOFF_SECONDS
270
+ end
271
+
272
+ # Close a PG connection we're discarding (the old conn at the top of a
273
+ # reconnect cycle, or a half-built one whose LISTEN raised). Best-effort.
274
+ def close_quietly(conn)
275
+ conn&.close if conn.respond_to?(:close)
276
+ rescue StandardError
277
+ nil
217
278
  end
218
279
 
219
280
  def safe_unlisten_all
220
281
  @listening_to.each do |channel|
221
- @conn.exec(%(UNLISTEN "#{channel}"))
282
+ # @conn may be nil if #stop interrupted the reconnect loop between
283
+ # closing the old connection and publishing a new one.
284
+ @conn&.exec(%(UNLISTEN "#{channel}"))
222
285
  rescue PG::Error
223
286
  # connection may be dead; nothing we can do
224
287
  end
data/lib/pgbus.rb CHANGED
@@ -20,6 +20,31 @@ module Pgbus
20
20
  class SchemaNotReady < Error; end
21
21
  class ReadTimeoutError < Error; end
22
22
 
23
+ # Raised by Client read paths when the in-memory connection-health circuit
24
+ # breaker (Client::ConnectionHealth) is open: the database has failed enough
25
+ # consecutive connection attempts that reads now fail fast without a pool
26
+ # checkout, sparing a dead database from the whole fleet re-polling and the
27
+ # error tracker from per-poll noise. Workers rescue this and idle until the
28
+ # breaker's backoff window admits a probe. See issue #197.
29
+ class ConnectionCircuitOpenError < Error; end
30
+
31
+ module Process
32
+ # A LISTEN connection landed on a read-only replica (pg_is_in_recovery() =>
33
+ # t). After a failover, stale DNS can point a fresh connection at the
34
+ # demoted master; NOTIFY fires only on the primary, so such a connection
35
+ # connects "successfully" but never wakes. Raised by
36
+ # Process::PrimaryValidator so the listener rejects it and retries with
37
+ # backoff (re-resolving DNS each attempt).
38
+ #
39
+ # Defined here rather than in process/primary_validator.rb because Zeitwerk
40
+ # requires each managed file to define exactly the constant matching its
41
+ # path — a second class in that file would break eager_load. lib/pgbus.rb
42
+ # is the (require-loaded) gem entry point, so it can define the error and
43
+ # explicitly namespace Process without conflicting with Zeitwerk's
44
+ # management of the lib/pgbus/process/ directory.
45
+ class ReplicaConnectionError < Error; end
46
+ end
47
+
23
48
  class << self
24
49
  # Process-global flag set by Worker#graceful_shutdown so the adapter
25
50
  # can report stopping? to ActiveJob::Continuation (Rails 8.1+).
@@ -35,6 +60,7 @@ module Pgbus
35
60
  loader.inflector.inflect(
36
61
  "pgbus" => "Pgbus",
37
62
  "cli" => "CLI",
63
+ "dlq" => "DLQ",
38
64
  "dsl" => "DSL",
39
65
  "capsule_dsl" => "CapsuleDSL",
40
66
  "mcp" => "MCP"
@@ -59,6 +85,12 @@ module Pgbus
59
85
  # root and tries to autoload Puma::Plugin, which collides with the real
60
86
  # Puma::Plugin class defined by the puma gem itself.
61
87
  loader.ignore("#{__dir__}/puma")
88
+ # lib/rubocop holds pgbus's custom RuboCop cops (Pgbus/NoRubyTimeout).
89
+ # They're loaded by RuboCop via .rubocop.yml's `require:`, not by the
90
+ # gem at runtime. Without this ignore, Zeitwerk scans lib/rubocop under
91
+ # the pgbus root and tries to autoload a `Rubocop` constant, colliding
92
+ # with the rubocop gem's real `RuboCop`.
93
+ loader.ignore("#{__dir__}/rubocop")
62
94
  loader
63
95
  end
64
96
  end
@@ -92,6 +124,11 @@ module Pgbus
92
124
 
93
125
  def configure
94
126
  yield configuration
127
+ # Fail loud at boot on an invalid value rather than leaving it dormant
128
+ # until a worker path consumes it. Checking the flag after the yield
129
+ # lets a block opt out for itself via `c.eager_validation = false`.
130
+ configuration.validate! if configuration.eager_validation
131
+ configuration
95
132
  end
96
133
 
97
134
  def client
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RuboCop
4
+ module Cop
5
+ module Pgbus
6
+ # Flags any use of Ruby's `Timeout.timeout`. It interrupts the target
7
+ # thread with `Thread#raise`, which can fire at an arbitrary point —
8
+ # including mid-libpq-call — and leave a pooled `PG::Connection` in a
9
+ # corrupt state that re-hangs or returns wrong results on reuse. Prefer a
10
+ # server-side/socket-level bound instead (Postgres `statement_timeout`,
11
+ # libpq `tcp_user_timeout`/`keepalives`, an explicit `IO`/socket timeout).
12
+ #
13
+ # There is no autocorrect on purpose: the safe replacement depends on what
14
+ # is being bounded, so a human must choose it.
15
+ #
16
+ # @example
17
+ # # bad
18
+ # Timeout.timeout(5) { do_work }
19
+ #
20
+ # # good — bound the actual resource (DB query, socket) directly
21
+ # conn.exec("SET statement_timeout = 5000")
22
+ # do_work
23
+ class NoRubyTimeout < Base
24
+ MSG = "Please be careful, Timeout is dangerous. Timeout.timeout uses " \
25
+ "Thread#raise and can corrupt a pooled connection mid-call. Bound the " \
26
+ "resource itself instead (statement_timeout / tcp_user_timeout / socket timeout)."
27
+
28
+ # Matches `Timeout.timeout(...)` and `::Timeout.timeout(...)`.
29
+ # @!method ruby_timeout?(node)
30
+ def_node_matcher :ruby_timeout?, <<~PATTERN
31
+ (send (const {nil? cbase} :Timeout) :timeout ...)
32
+ PATTERN
33
+
34
+ def on_send(node)
35
+ return unless ruby_timeout?(node)
36
+
37
+ add_offense(node.loc.selector)
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Entry point for pgbus's custom RuboCop cops. Required from .rubocop.yml via
4
+ # `require:` so the cops are available when linting this repo.
5
+ require_relative "cop/pgbus/no_ruby_timeout"
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ namespace :pgbus do
4
+ desc "Run environment diagnostics (config, DB, PGMQ, queues, LISTEN/NOTIFY, process liveness)"
5
+ task doctor: :environment do
6
+ require "pgbus/doctor"
7
+
8
+ doctor = Pgbus::Doctor.new
9
+ puts doctor.report
10
+ exit 1 unless doctor.success?
11
+ end
12
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pgbus
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.7
4
+ version: 0.9.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Mikael Henriksson
@@ -252,7 +252,9 @@ files:
252
252
  - lib/pgbus/bus_record.rb
253
253
  - lib/pgbus/circuit_breaker.rb
254
254
  - lib/pgbus/cli.rb
255
+ - lib/pgbus/cli/dlq.rb
255
256
  - lib/pgbus/client.rb
257
+ - lib/pgbus/client/connection_health.rb
256
258
  - lib/pgbus/client/ensure_stream_queue.rb
257
259
  - lib/pgbus/client/notify_stream.rb
258
260
  - lib/pgbus/client/read_after.rb
@@ -263,6 +265,7 @@ files:
263
265
  - lib/pgbus/configuration.rb
264
266
  - lib/pgbus/configuration/capsule_dsl.rb
265
267
  - lib/pgbus/dedup_cache.rb
268
+ - lib/pgbus/doctor.rb
266
269
  - lib/pgbus/engine.rb
267
270
  - lib/pgbus/error_reporter.rb
268
271
  - lib/pgbus/event.rb
@@ -305,6 +308,12 @@ files:
305
308
  - lib/pgbus/mcp/tools/recurring_tool.rb
306
309
  - lib/pgbus/mcp/tools/stats_tool.rb
307
310
  - lib/pgbus/mcp/tools/throughput_tool.rb
311
+ - lib/pgbus/metrics.rb
312
+ - lib/pgbus/metrics/backend.rb
313
+ - lib/pgbus/metrics/backends/prometheus.rb
314
+ - lib/pgbus/metrics/backends/statsd.rb
315
+ - lib/pgbus/metrics/prometheus_exporter.rb
316
+ - lib/pgbus/metrics/subscriber.rb
308
317
  - lib/pgbus/outbox.rb
309
318
  - lib/pgbus/outbox/poller.rb
310
319
  - lib/pgbus/pgmq_schema.rb
@@ -314,7 +323,10 @@ files:
314
323
  - lib/pgbus/process/dispatcher.rb
315
324
  - lib/pgbus/process/heartbeat.rb
316
325
  - lib/pgbus/process/lifecycle.rb
326
+ - lib/pgbus/process/memory_usage.rb
317
327
  - lib/pgbus/process/notify_listener.rb
328
+ - lib/pgbus/process/notify_probe.rb
329
+ - lib/pgbus/process/primary_validator.rb
318
330
  - lib/pgbus/process/queue_lock.rb
319
331
  - lib/pgbus/process/signal_handler.rb
320
332
  - lib/pgbus/process/supervisor.rb
@@ -355,6 +367,8 @@ files:
355
367
  - lib/pgbus/version.rb
356
368
  - lib/pgbus/web/authentication.rb
357
369
  - lib/pgbus/web/data_source.rb
370
+ - lib/pgbus/web/health_app.rb
371
+ - lib/pgbus/web/health_server.rb
358
372
  - lib/pgbus/web/metrics_serializer.rb
359
373
  - lib/pgbus/web/payload_filter.rb
360
374
  - lib/pgbus/web/stream_app.rb
@@ -369,7 +383,10 @@ files:
369
383
  - lib/pgbus/web/streamer/stream_counter.rb
370
384
  - lib/pgbus/web/streamer/stream_event_dispatcher.rb
371
385
  - lib/puma/plugin/pgbus_streams.rb
386
+ - lib/rubocop/cop/pgbus/no_ruby_timeout.rb
387
+ - lib/rubocop/pgbus.rb
372
388
  - lib/tasks/pgbus_autovacuum.rake
389
+ - lib/tasks/pgbus_doctor.rake
373
390
  - lib/tasks/pgbus_pgmq.rake
374
391
  - lib/tasks/pgbus_queues.rake
375
392
  - lib/tasks/pgbus_streams.rake