pgbus 0.9.7 → 0.9.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +38 -0
- data/README.md +119 -1
- data/Rakefile +10 -1
- data/app/helpers/pgbus/application_helper.rb +37 -0
- data/app/views/pgbus/processes/_processes_table.html.erb +4 -1
- data/config/locales/da.yml +4 -0
- data/config/locales/de.yml +4 -0
- data/config/locales/en.yml +4 -0
- data/config/locales/es.yml +4 -0
- data/config/locales/fi.yml +4 -0
- data/config/locales/fr.yml +4 -0
- data/config/locales/it.yml +4 -0
- data/config/locales/ja.yml +4 -0
- data/config/locales/nb.yml +4 -0
- data/config/locales/nl.yml +4 -0
- data/config/locales/pt.yml +4 -0
- data/config/locales/sv.yml +4 -0
- data/lib/pgbus/active_job/executor.rb +25 -4
- data/lib/pgbus/cli/dlq.rb +164 -0
- data/lib/pgbus/cli.rb +18 -1
- data/lib/pgbus/client/connection_health.rb +194 -0
- data/lib/pgbus/client.rb +592 -73
- data/lib/pgbus/config_loader.rb +23 -4
- data/lib/pgbus/configuration.rb +98 -12
- data/lib/pgbus/dedup_cache.rb +8 -0
- data/lib/pgbus/doctor.rb +250 -0
- data/lib/pgbus/engine.rb +15 -0
- data/lib/pgbus/execution_pools/async_pool.rb +7 -0
- data/lib/pgbus/execution_pools/thread_pool.rb +7 -0
- data/lib/pgbus/instrumentation.rb +1 -0
- data/lib/pgbus/integrations/appsignal/probe.rb +23 -1
- data/lib/pgbus/metrics/backend.rb +38 -0
- data/lib/pgbus/metrics/backends/prometheus.rb +123 -0
- data/lib/pgbus/metrics/backends/statsd.rb +64 -0
- data/lib/pgbus/metrics/prometheus_exporter.rb +34 -0
- data/lib/pgbus/metrics/subscriber.rb +190 -0
- data/lib/pgbus/metrics.rb +42 -0
- data/lib/pgbus/process/consumer.rb +215 -8
- data/lib/pgbus/process/consumer_priority.rb +34 -0
- data/lib/pgbus/process/dispatcher.rb +265 -41
- data/lib/pgbus/process/heartbeat.rb +18 -5
- data/lib/pgbus/process/memory_usage.rb +48 -0
- data/lib/pgbus/process/notify_listener.rb +26 -7
- data/lib/pgbus/process/notify_probe.rb +96 -0
- data/lib/pgbus/process/primary_validator.rb +53 -0
- data/lib/pgbus/process/signal_handler.rb +6 -0
- data/lib/pgbus/process/supervisor.rb +396 -46
- data/lib/pgbus/process/worker.rb +298 -35
- data/lib/pgbus/recurring/scheduler.rb +15 -1
- data/lib/pgbus/streams/turbo_broadcastable.rb +7 -5
- data/lib/pgbus/table_maintenance.rb +13 -2
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source.rb +20 -4
- data/lib/pgbus/web/health_app.rb +102 -0
- data/lib/pgbus/web/health_server.rb +144 -0
- data/lib/pgbus/web/streamer/instance.rb +55 -1
- data/lib/pgbus/web/streamer/listener.rb +72 -9
- data/lib/pgbus.rb +37 -0
- data/lib/rubocop/cop/pgbus/no_ruby_timeout.rb +42 -0
- data/lib/rubocop/pgbus.rb +5 -0
- data/lib/tasks/pgbus_doctor.rake +12 -0
- metadata +18 -1
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Pgbus
|
|
6
|
+
module Web
|
|
7
|
+
# Plain Rack app exposing HTTP liveness and readiness probes for
|
|
8
|
+
# orchestrators (Kubernetes kubelet, load balancers). Mount it in the host
|
|
9
|
+
# Rails app — no auth, no DB access on the liveness path:
|
|
10
|
+
#
|
|
11
|
+
# # config/routes.rb
|
|
12
|
+
# mount Pgbus::Web::HealthApp.new => "/pgbus/health"
|
|
13
|
+
#
|
|
14
|
+
# # kubelet
|
|
15
|
+
# livenessProbe: { httpGet: { path: /pgbus/health/livez, port: 3000 } }
|
|
16
|
+
# readinessProbe: { httpGet: { path: /pgbus/health/readyz, port: 3000 } }
|
|
17
|
+
#
|
|
18
|
+
# The supervisor process can also serve these two paths standalone via
|
|
19
|
+
# {Pgbus::Web::HealthServer} when `config.health_port` is set — that path
|
|
20
|
+
# synthesizes a minimal Rack env and calls this same #call, so the routing
|
|
21
|
+
# and verdict logic live in exactly one place.
|
|
22
|
+
#
|
|
23
|
+
# Routing (env-only, no Rack::Request so it works under the bare-socket
|
|
24
|
+
# HealthServer too):
|
|
25
|
+
# GET /livez → 200 text/plain "ok", unconditionally, NO database access
|
|
26
|
+
# (the serving process is up — that is all liveness means).
|
|
27
|
+
# GET /readyz → build a DataSource, run MCP::HealthAnalyzer#verdict:
|
|
28
|
+
# OK / DEGRADED → 200, STALLED → 503, body = verdict JSON.
|
|
29
|
+
# Any StandardError (DB unreachable) → 503 {"status":"ERROR"},
|
|
30
|
+
# logged via Pgbus.logger (never swallowed).
|
|
31
|
+
# unknown path → 404; non-GET on a known path → 405.
|
|
32
|
+
class HealthApp
|
|
33
|
+
LIVEZ = "/livez"
|
|
34
|
+
READYZ = "/readyz"
|
|
35
|
+
KNOWN_PATHS = [LIVEZ, READYZ].freeze
|
|
36
|
+
private_constant :KNOWN_PATHS
|
|
37
|
+
|
|
38
|
+
TEXT = { "Content-Type" => "text/plain" }.freeze
|
|
39
|
+
JSON_HEADERS = { "Content-Type" => "application/json" }.freeze
|
|
40
|
+
private_constant :TEXT, :JSON_HEADERS
|
|
41
|
+
|
|
42
|
+
# DEGRADED is deliberately "ready": the serving process can still answer,
|
|
43
|
+
# and flapping a pod out of rotation on a transient DEGRADED (a stale
|
|
44
|
+
# sibling, a paused queue) would remove capacity for no gain. Only STALLED
|
|
45
|
+
# — the silent-wedge signal — fails readiness.
|
|
46
|
+
READY_STATUSES = %w[OK DEGRADED].freeze
|
|
47
|
+
private_constant :READY_STATUSES
|
|
48
|
+
|
|
49
|
+
# @param data_source [Pgbus::Web::DataSource, nil] read layer for /readyz.
|
|
50
|
+
# nil (the default) builds a fresh DataSource per readiness check, which
|
|
51
|
+
# avoids serving stale metrics from a long-lived app's memoized instance.
|
|
52
|
+
def initialize(data_source: nil)
|
|
53
|
+
@data_source = data_source
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def call(env)
|
|
57
|
+
path = env["PATH_INFO"].to_s
|
|
58
|
+
return not_found unless KNOWN_PATHS.include?(path)
|
|
59
|
+
return method_not_allowed unless env["REQUEST_METHOD"] == "GET"
|
|
60
|
+
|
|
61
|
+
path == LIVEZ ? livez : readyz
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
private
|
|
65
|
+
|
|
66
|
+
# Liveness: the process is running. No DB, no analyzer, no config lookups.
|
|
67
|
+
def livez
|
|
68
|
+
[200, TEXT.dup, ["ok"]]
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def readyz
|
|
72
|
+
# HealthAnalyzer lives in the MCP namespace, which is excluded from
|
|
73
|
+
# Zeitwerk (its *tools* subclass the optional `mcp` gem). The analyzer
|
|
74
|
+
# itself has no gem dependency, so require just that one file — the
|
|
75
|
+
# standalone health server must work without the `mcp` gem installed.
|
|
76
|
+
require "pgbus/mcp/health_analyzer" unless defined?(Pgbus::MCP::HealthAnalyzer)
|
|
77
|
+
|
|
78
|
+
verdict = Pgbus::MCP::HealthAnalyzer.new(data_source_for_check).verdict
|
|
79
|
+
status = READY_STATUSES.include?(verdict[:status].to_s) ? 200 : 503
|
|
80
|
+
[status, JSON_HEADERS.dup, [verdict.to_json]]
|
|
81
|
+
rescue StandardError => e
|
|
82
|
+
Pgbus.logger.error { "[Pgbus::Web::HealthApp] readiness check failed: #{e.class}: #{e.message}" }
|
|
83
|
+
[503, JSON_HEADERS.dup, [{ status: "ERROR", error: e.message }.to_json]]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Reuse an injected DataSource (tests, an app that wants one shared
|
|
87
|
+
# instance); otherwise build a fresh one each check so per-instance
|
|
88
|
+
# memoization can never serve stale queue/process metrics.
|
|
89
|
+
def data_source_for_check
|
|
90
|
+
@data_source || DataSource.new
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def not_found
|
|
94
|
+
[404, TEXT.dup, ["pgbus: not found"]]
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def method_not_allowed
|
|
98
|
+
[405, TEXT.merge("Allow" => "GET"), ["pgbus: method not allowed"]]
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "socket"
|
|
4
|
+
|
|
5
|
+
module Pgbus
|
|
6
|
+
module Web
|
|
7
|
+
# Standalone HTTP health server for the supervisor process. When
|
|
8
|
+
# `config.health_port` is set, Supervisor#run starts one of these so a
|
|
9
|
+
# Kubernetes kubelet can probe the supervisor directly — the process that
|
|
10
|
+
# forks and watches workers — without the supervisor having to boot Rails,
|
|
11
|
+
# Puma, or the dashboard.
|
|
12
|
+
#
|
|
13
|
+
# It is intentionally tiny: one accept-loop thread over a TCPServer that
|
|
14
|
+
# reads only the HTTP request line (`GET /readyz HTTP/1.1`), synthesizes a
|
|
15
|
+
# minimal Rack env, and hands it to {HealthApp#call}. All routing and the
|
|
16
|
+
# OK/DEGRADED/STALLED → status mapping live in HealthApp, so the mounted-in-
|
|
17
|
+
# Rails path and this standalone path share exactly one implementation.
|
|
18
|
+
#
|
|
19
|
+
# This is a health probe surface, not a general HTTP server: it speaks just
|
|
20
|
+
# enough HTTP/1.0 to answer a kubelet. It only parses the request line
|
|
21
|
+
# (method + path), ignores headers and body, and closes the connection
|
|
22
|
+
# after each response (no keep-alive).
|
|
23
|
+
class HealthServer
|
|
24
|
+
# Cap the request line so a client can't stream unbounded data at the
|
|
25
|
+
# probe port. A real probe request line is well under 100 bytes.
|
|
26
|
+
MAX_REQUEST_LINE = 8_192
|
|
27
|
+
private_constant :MAX_REQUEST_LINE
|
|
28
|
+
|
|
29
|
+
STATUS_TEXT = {
|
|
30
|
+
200 => "OK", 404 => "Not Found", 405 => "Method Not Allowed", 503 => "Service Unavailable"
|
|
31
|
+
}.freeze
|
|
32
|
+
private_constant :STATUS_TEXT
|
|
33
|
+
|
|
34
|
+
# @return [Integer, nil] the bound port. Equals the configured port, or
|
|
35
|
+
# the OS-assigned port when the configured port was 0 (used in tests).
|
|
36
|
+
# nil before #start.
|
|
37
|
+
attr_reader :port
|
|
38
|
+
|
|
39
|
+
def initialize(port:, bind: "127.0.0.1", app: nil, logger: nil)
|
|
40
|
+
@configured_port = port
|
|
41
|
+
@bind = bind
|
|
42
|
+
@app = app || HealthApp.new
|
|
43
|
+
@logger = logger
|
|
44
|
+
@server = nil
|
|
45
|
+
@thread = nil
|
|
46
|
+
@port = nil
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Bind the port and start the accept loop. Idempotent: a second call
|
|
50
|
+
# while running is a no-op (so a supervisor restart path can't double-bind).
|
|
51
|
+
def start
|
|
52
|
+
return if @server
|
|
53
|
+
|
|
54
|
+
@server = TCPServer.new(@bind, @configured_port)
|
|
55
|
+
@port = @server.addr[1]
|
|
56
|
+
@thread = Thread.new { accept_loop }
|
|
57
|
+
logger.info { "[Pgbus::Web::HealthServer] listening on #{@bind}:#{@port} (/livez, /readyz)" }
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Close the listening socket (unblocking the accept loop, which then
|
|
61
|
+
# exits) and join the thread. Safe to call when never started or twice.
|
|
62
|
+
def stop
|
|
63
|
+
server = @server
|
|
64
|
+
@server = nil
|
|
65
|
+
close_socket(server)
|
|
66
|
+
@thread&.join(2)
|
|
67
|
+
@thread = nil
|
|
68
|
+
@port = nil
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
# accept blocks until stop closes the socket, which raises here and ends
|
|
74
|
+
# the loop. Any per-connection error is logged and swallowed so one bad
|
|
75
|
+
# client can never take the probe surface down.
|
|
76
|
+
def accept_loop
|
|
77
|
+
loop do
|
|
78
|
+
server = @server
|
|
79
|
+
break unless server
|
|
80
|
+
|
|
81
|
+
client =
|
|
82
|
+
begin
|
|
83
|
+
server.accept
|
|
84
|
+
rescue IOError, Errno::EBADF, Errno::EINVAL
|
|
85
|
+
break # socket closed by #stop
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
handle_client(client)
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def handle_client(client)
|
|
93
|
+
method, path = read_request_line(client)
|
|
94
|
+
return unless method
|
|
95
|
+
|
|
96
|
+
status, headers, body = @app.call(rack_env(method, path))
|
|
97
|
+
write_response(client, status, headers, body)
|
|
98
|
+
rescue StandardError => e
|
|
99
|
+
logger.warn { "[Pgbus::Web::HealthServer] connection error: #{e.class}: #{e.message}" }
|
|
100
|
+
ensure
|
|
101
|
+
close_socket(client)
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Read and parse only the first line: "METHOD PATH HTTP/x.y". Returns
|
|
105
|
+
# [method, path] or [nil, nil] when the line is missing/garbage so the
|
|
106
|
+
# caller drops the connection without dispatching.
|
|
107
|
+
def read_request_line(client)
|
|
108
|
+
line = client.gets("\r\n", MAX_REQUEST_LINE)
|
|
109
|
+
return [nil, nil] if line.nil?
|
|
110
|
+
|
|
111
|
+
method, path, = line.strip.split(" ", 3)
|
|
112
|
+
return [nil, nil] if method.nil? || path.nil?
|
|
113
|
+
|
|
114
|
+
[method, path]
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Minimal Rack env — HealthApp reads only REQUEST_METHOD and PATH_INFO.
|
|
118
|
+
# Strip any query string from the path so PATH_INFO stays a bare path.
|
|
119
|
+
def rack_env(method, path)
|
|
120
|
+
{ "REQUEST_METHOD" => method, "PATH_INFO" => path.split("?", 2).first }
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def write_response(client, status, headers, body)
|
|
124
|
+
reason = STATUS_TEXT.fetch(status, "OK")
|
|
125
|
+
payload = Array(body).join
|
|
126
|
+
lines = ["HTTP/1.0 #{status} #{reason}"]
|
|
127
|
+
headers.each { |k, v| lines << "#{k}: #{v}" }
|
|
128
|
+
lines << "Content-Length: #{payload.bytesize}"
|
|
129
|
+
lines << "Connection: close"
|
|
130
|
+
client.write("#{lines.join("\r\n")}\r\n\r\n#{payload}")
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def close_socket(socket)
|
|
134
|
+
socket&.close
|
|
135
|
+
rescue IOError, Errno::EBADF
|
|
136
|
+
nil
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def logger
|
|
140
|
+
@logger || Pgbus.logger
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
end
|
|
@@ -42,7 +42,13 @@ module Pgbus
|
|
|
42
42
|
pg_connection: @pg_connection,
|
|
43
43
|
dispatch_queue: @dispatch_queue,
|
|
44
44
|
health_check_ms: @config.streams_listen_health_check_ms,
|
|
45
|
-
logger: @logger
|
|
45
|
+
logger: @logger,
|
|
46
|
+
# On reconnect the Listener rebuilds its OWN connection via this
|
|
47
|
+
# factory (fresh connect re-resolves DNS, converges on the promoted
|
|
48
|
+
# primary after a failover) instead of resetting a possibly-dead
|
|
49
|
+
# socket. Skipped for an injected pg_connection: (unit tests) — a
|
|
50
|
+
# nil factory falls back to conn.reset.
|
|
51
|
+
connection_factory: pg_connection ? nil : -> { build_raw_pg_connection }
|
|
46
52
|
)
|
|
47
53
|
@dispatcher = StreamEventDispatcher.new(
|
|
48
54
|
client: @client,
|
|
@@ -143,7 +149,20 @@ module Pgbus
|
|
|
143
149
|
end
|
|
144
150
|
end
|
|
145
151
|
|
|
152
|
+
# Build the INITIAL LISTEN connection: raw connect, reject a replica
|
|
153
|
+
# fatally (a misconfiguration at boot), then run the one-shot delivery
|
|
154
|
+
# self-probe. The reconnect path uses build_raw_pg_connection directly
|
|
155
|
+
# (validation retries there, probe is skipped) — see the factory wired
|
|
156
|
+
# into Listener.new above.
|
|
146
157
|
def build_pg_connection
|
|
158
|
+
probe(validate_primary(build_raw_pg_connection))
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# Raw PG.connect with no validation or probe. This is what the Listener's
|
|
162
|
+
# connection_factory calls on every reconnect attempt: the reconnect loop
|
|
163
|
+
# runs its own PrimaryValidator (retrying on a replica) and skips the
|
|
164
|
+
# probe to stay cheap, mirroring Pgbus::Process::NotifyListener.
|
|
165
|
+
def build_raw_pg_connection
|
|
147
166
|
require "pg" unless defined?(::PG::Connection)
|
|
148
167
|
opts = @config.streams_connection_options
|
|
149
168
|
case opts
|
|
@@ -167,6 +186,41 @@ module Pgbus
|
|
|
167
186
|
"Cannot build streamer PG connection from #{opts.class}"
|
|
168
187
|
end
|
|
169
188
|
end
|
|
189
|
+
|
|
190
|
+
# Reject a connection that landed on a read-only replica before the
|
|
191
|
+
# streamer wires it into a Listener. After a failover, stale DNS can
|
|
192
|
+
# point this fresh PG.connect at the demoted master; NOTIFY fires only
|
|
193
|
+
# on the primary, so the streamer would sit deaf. Unlike the reconnect
|
|
194
|
+
# loop (which retries), this runs once at Instance construction (Puma
|
|
195
|
+
# worker boot), so a replica here is a fatal misconfiguration: re-raise
|
|
196
|
+
# as a ConfigurationError naming the direct-connection overrides.
|
|
197
|
+
def validate_primary(pg_connection)
|
|
198
|
+
Pgbus::Process::PrimaryValidator.validate_primary!(pg_connection)
|
|
199
|
+
rescue Pgbus::Process::ReplicaConnectionError => e
|
|
200
|
+
close_quietly(pg_connection)
|
|
201
|
+
raise Pgbus::ConfigurationError,
|
|
202
|
+
"Streamer LISTEN connection landed on a read-only replica " \
|
|
203
|
+
"(#{e.message}). NOTIFY fires only on the primary, so the " \
|
|
204
|
+
"streamer would never wake. Point the streamer at a DIRECT " \
|
|
205
|
+
"primary via streams_database_url (or streams_host / " \
|
|
206
|
+
"streams_port)."
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def close_quietly(pg_connection)
|
|
210
|
+
pg_connection.close if pg_connection.respond_to?(:close)
|
|
211
|
+
rescue StandardError
|
|
212
|
+
nil
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# One-shot LISTEN/NOTIFY delivery self-probe on the freshly built
|
|
216
|
+
# connection. A transaction-mode pooler or replica that silently breaks
|
|
217
|
+
# NOTIFY is surfaced with an actionable error (naming the streams_*
|
|
218
|
+
# overrides); the streamer still starts and degrades to slow SSE
|
|
219
|
+
# updates rather than crashing. Returns the connection unchanged.
|
|
220
|
+
def probe(pg_connection)
|
|
221
|
+
Pgbus::Process::NotifyProbe.probe_notify_delivery!(pg_connection, logger: @logger)
|
|
222
|
+
pg_connection
|
|
223
|
+
end
|
|
170
224
|
end
|
|
171
225
|
end
|
|
172
226
|
end
|
|
@@ -38,13 +38,21 @@ module Pgbus
|
|
|
38
38
|
CHANNEL_PREFIX = "pgmq.q_"
|
|
39
39
|
CHANNEL_SUFFIX = ".INSERT"
|
|
40
40
|
|
|
41
|
+
RECONNECT_BACKOFF_SECONDS = 0.5
|
|
42
|
+
|
|
41
43
|
attr_reader :listening_to
|
|
42
44
|
|
|
43
|
-
|
|
45
|
+
# @param connection_factory [#call, nil] builds a FRESH PG connection on
|
|
46
|
+
# each reconnect attempt (the Instance passes -> { build_pg_connection }).
|
|
47
|
+
# When nil (tests injecting only pg_connection:), reconnect falls back to
|
|
48
|
+
# the legacy single-shot @conn.reset.
|
|
49
|
+
def initialize(pg_connection:, dispatch_queue:, health_check_ms:,
|
|
50
|
+
logger: Pgbus.logger, connection_factory: nil)
|
|
44
51
|
@conn = pg_connection
|
|
45
52
|
@dispatch_queue = dispatch_queue
|
|
46
53
|
@health_check_ms = health_check_ms
|
|
47
54
|
@logger = logger
|
|
55
|
+
@connection_factory = connection_factory
|
|
48
56
|
@listening_to = Set.new
|
|
49
57
|
@commands = Queue.new
|
|
50
58
|
@running = false
|
|
@@ -197,13 +205,58 @@ module Pgbus
|
|
|
197
205
|
@conn.exec("SELECT 1")
|
|
198
206
|
end
|
|
199
207
|
|
|
208
|
+
# Retry reconnect until we succeed (fresh conn + every channel
|
|
209
|
+
# re-LISTENed) or #stop flips @running to false. Mirrors
|
|
210
|
+
# NotifyListener#reconnect!: a single failed attempt never returns with
|
|
211
|
+
# a dead connection while running, so ensure_listening acks can't
|
|
212
|
+
# succeed vacuously against a broken LISTEN socket and strand SSE
|
|
213
|
+
# clients. When no connection_factory was injected (tests passing only
|
|
214
|
+
# pg_connection:), fall back to the legacy single-shot conn.reset.
|
|
200
215
|
def reconnect!
|
|
216
|
+
return reconnect_via_reset unless @connection_factory
|
|
217
|
+
|
|
218
|
+
# Snapshot the canonical subscription set BEFORE tearing down the old
|
|
219
|
+
# connection. We rebuild @listening_to from this and only publish it
|
|
220
|
+
# once every channel has been re-LISTENed on the new conn, so a
|
|
221
|
+
# mid-loop LISTEN failure never loses channels.
|
|
222
|
+
channels = @listening_to.to_a
|
|
223
|
+
loop do
|
|
224
|
+
return unless @running
|
|
225
|
+
|
|
226
|
+
close_quietly(@conn)
|
|
227
|
+
@conn = nil
|
|
228
|
+
new_conn = nil
|
|
229
|
+
begin
|
|
230
|
+
new_conn = @connection_factory.call
|
|
231
|
+
# Reject a replica before re-LISTENing. A fresh connect re-resolves
|
|
232
|
+
# DNS, so backing off and retrying converges on the promoted primary
|
|
233
|
+
# once DNS catches up after a failover. NOTIFY fires only on the
|
|
234
|
+
# primary; re-LISTENing on a replica registers channels that never wake.
|
|
235
|
+
Pgbus::Process::PrimaryValidator.validate_primary!(new_conn)
|
|
236
|
+
channels.each { |channel| new_conn.exec(%(LISTEN "#{channel}")) }
|
|
237
|
+
rescue PG::Error, Pgbus::Process::ReplicaConnectionError => e
|
|
238
|
+
# The factory may have returned a live conn before a later LISTEN
|
|
239
|
+
# raised (or validate_primary! rejected a replica). Close the
|
|
240
|
+
# partial conn so repeated failures don't leak PG connections.
|
|
241
|
+
close_quietly(new_conn)
|
|
242
|
+
@logger.error { "[Pgbus::Streamer::Listener] reconnect failed: #{e.class}: #{e.message}" }
|
|
243
|
+
sleep RECONNECT_BACKOFF_SECONDS
|
|
244
|
+
next
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
@conn = new_conn
|
|
248
|
+
@listening_to = Set.new(channels)
|
|
249
|
+
return
|
|
250
|
+
end
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
# Legacy fallback for tests that inject only pg_connection: (no factory).
|
|
254
|
+
# A single conn.reset attempt; on failure it logs and backs off, leaving
|
|
255
|
+
# recovery to the next run_loop cycle. Kept for backwards compatibility
|
|
256
|
+
# with test wiring; production always injects a connection_factory.
|
|
257
|
+
def reconnect_via_reset
|
|
201
258
|
@conn.reset
|
|
202
|
-
|
|
203
|
-
# mid-loop LISTEN raises, we keep the original set so the
|
|
204
|
-
# next reconnect cycle still knows which channels need to
|
|
205
|
-
# come back. The previous version cleared first and lost
|
|
206
|
-
# any channels not yet retried on a transient error.
|
|
259
|
+
Pgbus::Process::PrimaryValidator.validate_primary!(@conn)
|
|
207
260
|
to_relisten = @listening_to.to_a
|
|
208
261
|
new_listening = Set.new
|
|
209
262
|
to_relisten.each do |channel|
|
|
@@ -211,14 +264,24 @@ module Pgbus
|
|
|
211
264
|
new_listening.add(channel)
|
|
212
265
|
end
|
|
213
266
|
@listening_to = new_listening
|
|
214
|
-
rescue PG::Error => e
|
|
267
|
+
rescue PG::Error, Pgbus::Process::ReplicaConnectionError => e
|
|
215
268
|
@logger.error { "[Pgbus::Streamer::Listener] reconnect failed: #{e.class}: #{e.message}" }
|
|
216
|
-
sleep
|
|
269
|
+
sleep RECONNECT_BACKOFF_SECONDS
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
# Close a PG connection we're discarding (the old conn at the top of a
|
|
273
|
+
# reconnect cycle, or a half-built one whose LISTEN raised). Best-effort.
|
|
274
|
+
def close_quietly(conn)
|
|
275
|
+
conn&.close if conn.respond_to?(:close)
|
|
276
|
+
rescue StandardError
|
|
277
|
+
nil
|
|
217
278
|
end
|
|
218
279
|
|
|
219
280
|
def safe_unlisten_all
|
|
220
281
|
@listening_to.each do |channel|
|
|
221
|
-
@conn
|
|
282
|
+
# @conn may be nil if #stop interrupted the reconnect loop between
|
|
283
|
+
# closing the old connection and publishing a new one.
|
|
284
|
+
@conn&.exec(%(UNLISTEN "#{channel}"))
|
|
222
285
|
rescue PG::Error
|
|
223
286
|
# connection may be dead; nothing we can do
|
|
224
287
|
end
|
data/lib/pgbus.rb
CHANGED
|
@@ -20,6 +20,31 @@ module Pgbus
|
|
|
20
20
|
class SchemaNotReady < Error; end
|
|
21
21
|
class ReadTimeoutError < Error; end
|
|
22
22
|
|
|
23
|
+
# Raised by Client read paths when the in-memory connection-health circuit
|
|
24
|
+
# breaker (Client::ConnectionHealth) is open: the database has failed enough
|
|
25
|
+
# consecutive connection attempts that reads now fail fast without a pool
|
|
26
|
+
# checkout, sparing a dead database from the whole fleet re-polling and the
|
|
27
|
+
# error tracker from per-poll noise. Workers rescue this and idle until the
|
|
28
|
+
# breaker's backoff window admits a probe. See issue #197.
|
|
29
|
+
class ConnectionCircuitOpenError < Error; end
|
|
30
|
+
|
|
31
|
+
module Process
|
|
32
|
+
# A LISTEN connection landed on a read-only replica (pg_is_in_recovery() =>
|
|
33
|
+
# t). After a failover, stale DNS can point a fresh connection at the
|
|
34
|
+
# demoted master; NOTIFY fires only on the primary, so such a connection
|
|
35
|
+
# connects "successfully" but never wakes. Raised by
|
|
36
|
+
# Process::PrimaryValidator so the listener rejects it and retries with
|
|
37
|
+
# backoff (re-resolving DNS each attempt).
|
|
38
|
+
#
|
|
39
|
+
# Defined here rather than in process/primary_validator.rb because Zeitwerk
|
|
40
|
+
# requires each managed file to define exactly the constant matching its
|
|
41
|
+
# path — a second class in that file would break eager_load. lib/pgbus.rb
|
|
42
|
+
# is the (require-loaded) gem entry point, so it can define the error and
|
|
43
|
+
# explicitly namespace Process without conflicting with Zeitwerk's
|
|
44
|
+
# management of the lib/pgbus/process/ directory.
|
|
45
|
+
class ReplicaConnectionError < Error; end
|
|
46
|
+
end
|
|
47
|
+
|
|
23
48
|
class << self
|
|
24
49
|
# Process-global flag set by Worker#graceful_shutdown so the adapter
|
|
25
50
|
# can report stopping? to ActiveJob::Continuation (Rails 8.1+).
|
|
@@ -35,6 +60,7 @@ module Pgbus
|
|
|
35
60
|
loader.inflector.inflect(
|
|
36
61
|
"pgbus" => "Pgbus",
|
|
37
62
|
"cli" => "CLI",
|
|
63
|
+
"dlq" => "DLQ",
|
|
38
64
|
"dsl" => "DSL",
|
|
39
65
|
"capsule_dsl" => "CapsuleDSL",
|
|
40
66
|
"mcp" => "MCP"
|
|
@@ -59,6 +85,12 @@ module Pgbus
|
|
|
59
85
|
# root and tries to autoload Puma::Plugin, which collides with the real
|
|
60
86
|
# Puma::Plugin class defined by the puma gem itself.
|
|
61
87
|
loader.ignore("#{__dir__}/puma")
|
|
88
|
+
# lib/rubocop holds pgbus's custom RuboCop cops (Pgbus/NoRubyTimeout).
|
|
89
|
+
# They're loaded by RuboCop via .rubocop.yml's `require:`, not by the
|
|
90
|
+
# gem at runtime. Without this ignore, Zeitwerk scans lib/rubocop under
|
|
91
|
+
# the pgbus root and tries to autoload a `Rubocop` constant, colliding
|
|
92
|
+
# with the rubocop gem's real `RuboCop`.
|
|
93
|
+
loader.ignore("#{__dir__}/rubocop")
|
|
62
94
|
loader
|
|
63
95
|
end
|
|
64
96
|
end
|
|
@@ -92,6 +124,11 @@ module Pgbus
|
|
|
92
124
|
|
|
93
125
|
def configure
|
|
94
126
|
yield configuration
|
|
127
|
+
# Fail loud at boot on an invalid value rather than leaving it dormant
|
|
128
|
+
# until a worker path consumes it. Checking the flag after the yield
|
|
129
|
+
# lets a block opt out for itself via `c.eager_validation = false`.
|
|
130
|
+
configuration.validate! if configuration.eager_validation
|
|
131
|
+
configuration
|
|
95
132
|
end
|
|
96
133
|
|
|
97
134
|
def client
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RuboCop
|
|
4
|
+
module Cop
|
|
5
|
+
module Pgbus
|
|
6
|
+
# Flags any use of Ruby's `Timeout.timeout`. It interrupts the target
|
|
7
|
+
# thread with `Thread#raise`, which can fire at an arbitrary point —
|
|
8
|
+
# including mid-libpq-call — and leave a pooled `PG::Connection` in a
|
|
9
|
+
# corrupt state that re-hangs or returns wrong results on reuse. Prefer a
|
|
10
|
+
# server-side/socket-level bound instead (Postgres `statement_timeout`,
|
|
11
|
+
# libpq `tcp_user_timeout`/`keepalives`, an explicit `IO`/socket timeout).
|
|
12
|
+
#
|
|
13
|
+
# There is no autocorrect on purpose: the safe replacement depends on what
|
|
14
|
+
# is being bounded, so a human must choose it.
|
|
15
|
+
#
|
|
16
|
+
# @example
|
|
17
|
+
# # bad
|
|
18
|
+
# Timeout.timeout(5) { do_work }
|
|
19
|
+
#
|
|
20
|
+
# # good — bound the actual resource (DB query, socket) directly
|
|
21
|
+
# conn.exec("SET statement_timeout = 5000")
|
|
22
|
+
# do_work
|
|
23
|
+
class NoRubyTimeout < Base
|
|
24
|
+
MSG = "Please be careful, Timeout is dangerous. Timeout.timeout uses " \
|
|
25
|
+
"Thread#raise and can corrupt a pooled connection mid-call. Bound the " \
|
|
26
|
+
"resource itself instead (statement_timeout / tcp_user_timeout / socket timeout)."
|
|
27
|
+
|
|
28
|
+
# Matches `Timeout.timeout(...)` and `::Timeout.timeout(...)`.
|
|
29
|
+
# @!method ruby_timeout?(node)
|
|
30
|
+
def_node_matcher :ruby_timeout?, <<~PATTERN
|
|
31
|
+
(send (const {nil? cbase} :Timeout) :timeout ...)
|
|
32
|
+
PATTERN
|
|
33
|
+
|
|
34
|
+
def on_send(node)
|
|
35
|
+
return unless ruby_timeout?(node)
|
|
36
|
+
|
|
37
|
+
add_offense(node.loc.selector)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
namespace :pgbus do
|
|
4
|
+
desc "Run environment diagnostics (config, DB, PGMQ, queues, LISTEN/NOTIFY, process liveness)"
|
|
5
|
+
task doctor: :environment do
|
|
6
|
+
require "pgbus/doctor"
|
|
7
|
+
|
|
8
|
+
doctor = Pgbus::Doctor.new
|
|
9
|
+
puts doctor.report
|
|
10
|
+
exit 1 unless doctor.success?
|
|
11
|
+
end
|
|
12
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pgbus
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.9.
|
|
4
|
+
version: 0.9.8
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Mikael Henriksson
|
|
@@ -252,7 +252,9 @@ files:
|
|
|
252
252
|
- lib/pgbus/bus_record.rb
|
|
253
253
|
- lib/pgbus/circuit_breaker.rb
|
|
254
254
|
- lib/pgbus/cli.rb
|
|
255
|
+
- lib/pgbus/cli/dlq.rb
|
|
255
256
|
- lib/pgbus/client.rb
|
|
257
|
+
- lib/pgbus/client/connection_health.rb
|
|
256
258
|
- lib/pgbus/client/ensure_stream_queue.rb
|
|
257
259
|
- lib/pgbus/client/notify_stream.rb
|
|
258
260
|
- lib/pgbus/client/read_after.rb
|
|
@@ -263,6 +265,7 @@ files:
|
|
|
263
265
|
- lib/pgbus/configuration.rb
|
|
264
266
|
- lib/pgbus/configuration/capsule_dsl.rb
|
|
265
267
|
- lib/pgbus/dedup_cache.rb
|
|
268
|
+
- lib/pgbus/doctor.rb
|
|
266
269
|
- lib/pgbus/engine.rb
|
|
267
270
|
- lib/pgbus/error_reporter.rb
|
|
268
271
|
- lib/pgbus/event.rb
|
|
@@ -305,6 +308,12 @@ files:
|
|
|
305
308
|
- lib/pgbus/mcp/tools/recurring_tool.rb
|
|
306
309
|
- lib/pgbus/mcp/tools/stats_tool.rb
|
|
307
310
|
- lib/pgbus/mcp/tools/throughput_tool.rb
|
|
311
|
+
- lib/pgbus/metrics.rb
|
|
312
|
+
- lib/pgbus/metrics/backend.rb
|
|
313
|
+
- lib/pgbus/metrics/backends/prometheus.rb
|
|
314
|
+
- lib/pgbus/metrics/backends/statsd.rb
|
|
315
|
+
- lib/pgbus/metrics/prometheus_exporter.rb
|
|
316
|
+
- lib/pgbus/metrics/subscriber.rb
|
|
308
317
|
- lib/pgbus/outbox.rb
|
|
309
318
|
- lib/pgbus/outbox/poller.rb
|
|
310
319
|
- lib/pgbus/pgmq_schema.rb
|
|
@@ -314,7 +323,10 @@ files:
|
|
|
314
323
|
- lib/pgbus/process/dispatcher.rb
|
|
315
324
|
- lib/pgbus/process/heartbeat.rb
|
|
316
325
|
- lib/pgbus/process/lifecycle.rb
|
|
326
|
+
- lib/pgbus/process/memory_usage.rb
|
|
317
327
|
- lib/pgbus/process/notify_listener.rb
|
|
328
|
+
- lib/pgbus/process/notify_probe.rb
|
|
329
|
+
- lib/pgbus/process/primary_validator.rb
|
|
318
330
|
- lib/pgbus/process/queue_lock.rb
|
|
319
331
|
- lib/pgbus/process/signal_handler.rb
|
|
320
332
|
- lib/pgbus/process/supervisor.rb
|
|
@@ -355,6 +367,8 @@ files:
|
|
|
355
367
|
- lib/pgbus/version.rb
|
|
356
368
|
- lib/pgbus/web/authentication.rb
|
|
357
369
|
- lib/pgbus/web/data_source.rb
|
|
370
|
+
- lib/pgbus/web/health_app.rb
|
|
371
|
+
- lib/pgbus/web/health_server.rb
|
|
358
372
|
- lib/pgbus/web/metrics_serializer.rb
|
|
359
373
|
- lib/pgbus/web/payload_filter.rb
|
|
360
374
|
- lib/pgbus/web/stream_app.rb
|
|
@@ -369,7 +383,10 @@ files:
|
|
|
369
383
|
- lib/pgbus/web/streamer/stream_counter.rb
|
|
370
384
|
- lib/pgbus/web/streamer/stream_event_dispatcher.rb
|
|
371
385
|
- lib/puma/plugin/pgbus_streams.rb
|
|
386
|
+
- lib/rubocop/cop/pgbus/no_ruby_timeout.rb
|
|
387
|
+
- lib/rubocop/pgbus.rb
|
|
372
388
|
- lib/tasks/pgbus_autovacuum.rake
|
|
389
|
+
- lib/tasks/pgbus_doctor.rake
|
|
373
390
|
- lib/tasks/pgbus_pgmq.rake
|
|
374
391
|
- lib/tasks/pgbus_queues.rake
|
|
375
392
|
- lib/tasks/pgbus_streams.rake
|