pgbus 0.9.7 → 0.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +367 -0
- data/README.md +454 -25
- data/Rakefile +10 -1
- data/app/helpers/pgbus/application_helper.rb +37 -0
- data/app/views/pgbus/processes/_processes_table.html.erb +4 -1
- data/config/locales/da.yml +4 -0
- data/config/locales/de.yml +4 -0
- data/config/locales/en.yml +4 -0
- data/config/locales/es.yml +4 -0
- data/config/locales/fi.yml +4 -0
- data/config/locales/fr.yml +4 -0
- data/config/locales/it.yml +4 -0
- data/config/locales/ja.yml +4 -0
- data/config/locales/nb.yml +4 -0
- data/config/locales/nl.yml +4 -0
- data/config/locales/pt.yml +4 -0
- data/config/locales/sv.yml +4 -0
- data/lib/generators/pgbus/{add_job_locks_generator.rb → add_uniqueness_keys_generator.rb} +5 -5
- data/lib/pgbus/active_job/adapter.rb +1 -1
- data/lib/pgbus/active_job/executor.rb +25 -4
- data/lib/pgbus/cli/dlq.rb +164 -0
- data/lib/pgbus/cli.rb +18 -1
- data/lib/pgbus/client/connection_health.rb +194 -0
- data/lib/pgbus/client.rb +592 -73
- data/lib/pgbus/config_loader.rb +50 -4
- data/lib/pgbus/configuration.rb +294 -79
- data/lib/pgbus/dedup_cache.rb +8 -0
- data/lib/pgbus/doctor.rb +275 -0
- data/lib/pgbus/engine.rb +15 -0
- data/lib/pgbus/execution_pools/async_pool.rb +9 -2
- data/lib/pgbus/execution_pools/thread_pool.rb +7 -0
- data/lib/pgbus/generators/config_converter.rb +7 -5
- data/lib/pgbus/generators/migration_detector.rb +20 -16
- data/lib/pgbus/instrumentation.rb +3 -1
- data/lib/pgbus/integrations/appsignal/probe.rb +23 -1
- data/lib/pgbus/integrations/appsignal/subscriber.rb +17 -2
- data/lib/pgbus/metrics/backend.rb +38 -0
- data/lib/pgbus/metrics/backends/prometheus.rb +123 -0
- data/lib/pgbus/metrics/backends/statsd.rb +64 -0
- data/lib/pgbus/metrics/prometheus_exporter.rb +34 -0
- data/lib/pgbus/metrics/subscriber.rb +214 -0
- data/lib/pgbus/metrics.rb +43 -0
- data/lib/pgbus/pgmq_schema/pgmq_v1.11.1.sql +2126 -0
- data/lib/pgbus/pgmq_schema.rb +7 -2
- data/lib/pgbus/process/consumer.rb +354 -18
- data/lib/pgbus/process/consumer_priority.rb +34 -0
- data/lib/pgbus/process/dispatcher.rb +265 -41
- data/lib/pgbus/process/heartbeat.rb +18 -5
- data/lib/pgbus/process/memory_usage.rb +48 -0
- data/lib/pgbus/process/notify_listener.rb +26 -7
- data/lib/pgbus/process/notify_probe.rb +96 -0
- data/lib/pgbus/process/primary_validator.rb +53 -0
- data/lib/pgbus/process/signal_handler.rb +6 -0
- data/lib/pgbus/process/supervisor.rb +423 -50
- data/lib/pgbus/process/worker.rb +288 -35
- data/lib/pgbus/recurring/schedule.rb +1 -2
- data/lib/pgbus/recurring/scheduler.rb +15 -1
- data/lib/pgbus/serializer.rb +4 -4
- data/lib/pgbus/streams/broadcastable_override.rb +0 -8
- data/lib/pgbus/streams/signed_name.rb +2 -2
- data/lib/pgbus/streams/turbo_broadcastable.rb +7 -5
- data/lib/pgbus/table_maintenance.rb +13 -2
- data/lib/pgbus/uniqueness.rb +11 -12
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source.rb +36 -4
- data/lib/pgbus/web/health_app.rb +102 -0
- data/lib/pgbus/web/health_server.rb +144 -0
- data/lib/pgbus/web/payload_filter.rb +3 -3
- data/lib/pgbus/web/streamer/instance.rb +58 -2
- data/lib/pgbus/web/streamer/listener.rb +69 -21
- data/lib/pgbus.rb +77 -0
- data/lib/tasks/pgbus_doctor.rake +12 -0
- metadata +19 -4
- data/app/models/pgbus/job_lock.rb +0 -98
- data/lib/generators/pgbus/templates/add_job_locks.rb.erb +0 -21
|
@@ -1234,18 +1234,50 @@ module Pgbus
|
|
|
1234
1234
|
|
|
1235
1235
|
def derive_process_status(stale, metadata, kind)
|
|
1236
1236
|
return :stale if stale
|
|
1237
|
-
return :healthy unless
|
|
1237
|
+
return :healthy unless metadata.is_a?(Hash)
|
|
1238
|
+
|
|
1239
|
+
threshold = stall_threshold_for(kind)
|
|
1240
|
+
return :healthy unless threshold
|
|
1238
1241
|
|
|
1239
1242
|
loop_tick = metadata["loop_tick_at"]
|
|
1240
1243
|
return :healthy unless loop_tick
|
|
1241
1244
|
|
|
1242
|
-
threshold = Pgbus.configuration.stall_threshold
|
|
1243
|
-
return :healthy unless threshold&.positive?
|
|
1244
|
-
|
|
1245
1245
|
age = Time.now.to_f - loop_tick.to_f
|
|
1246
1246
|
age > threshold ? :stalled : :healthy
|
|
1247
1247
|
end
|
|
1248
1248
|
|
|
1249
|
+
# Kind-aware stall threshold. Workers run a tight loop, so the base
|
|
1250
|
+
# stall_threshold applies directly. The dispatcher and scheduler sleep
|
|
1251
|
+
# between iterations, so their beacon is naturally up to one sleep
|
|
1252
|
+
# interval stale even when healthy — widen the window by that interval to
|
|
1253
|
+
# avoid false stalls. Consumers idle on an empty-read wait that can reach
|
|
1254
|
+
# NOTIFY_FALLBACK_POLL_SECONDS (15s) when a live listener drives wake-up,
|
|
1255
|
+
# so widen by that max wait when notify is on, or by the poll interval
|
|
1256
|
+
# otherwise. Kinds with no loop beacon (nil) are never stalled.
|
|
1257
|
+
def stall_threshold_for(kind)
|
|
1258
|
+
config = Pgbus.configuration
|
|
1259
|
+
base = config.stall_threshold
|
|
1260
|
+
return nil unless base&.positive?
|
|
1261
|
+
|
|
1262
|
+
case kind
|
|
1263
|
+
when "worker" then base
|
|
1264
|
+
when "dispatcher" then base + config.dispatch_interval
|
|
1265
|
+
when "scheduler" then base + config.recurring_schedule_interval
|
|
1266
|
+
when "consumer" then base + consumer_stall_slack(config)
|
|
1267
|
+
end
|
|
1268
|
+
end
|
|
1269
|
+
|
|
1270
|
+
# Widen the consumer's stall window by its maximum empty-read wait. With
|
|
1271
|
+
# notify wake-up live, that wait rises to NOTIFY_FALLBACK_POLL_SECONDS as a
|
|
1272
|
+
# safety net; without it, the consumer polls at polling_interval.
|
|
1273
|
+
def consumer_stall_slack(config)
|
|
1274
|
+
if config.worker_notify_wakeup?
|
|
1275
|
+
[config.polling_interval, Pgbus::Process::Consumer::NOTIFY_FALLBACK_POLL_SECONDS].max
|
|
1276
|
+
else
|
|
1277
|
+
config.polling_interval
|
|
1278
|
+
end
|
|
1279
|
+
end
|
|
1280
|
+
|
|
1249
1281
|
def format_batch(record)
|
|
1250
1282
|
total = record.total_jobs
|
|
1251
1283
|
done = record.completed_jobs + record.discarded_jobs
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Pgbus
|
|
6
|
+
module Web
|
|
7
|
+
# Plain Rack app exposing HTTP liveness and readiness probes for
|
|
8
|
+
# orchestrators (Kubernetes kubelet, load balancers). Mount it in the host
|
|
9
|
+
# Rails app — no auth, no DB access on the liveness path:
|
|
10
|
+
#
|
|
11
|
+
# # config/routes.rb
|
|
12
|
+
# mount Pgbus::Web::HealthApp.new => "/pgbus/health"
|
|
13
|
+
#
|
|
14
|
+
# # kubelet
|
|
15
|
+
# livenessProbe: { httpGet: { path: /pgbus/health/livez, port: 3000 } }
|
|
16
|
+
# readinessProbe: { httpGet: { path: /pgbus/health/readyz, port: 3000 } }
|
|
17
|
+
#
|
|
18
|
+
# The supervisor process can also serve these two paths standalone via
|
|
19
|
+
# {Pgbus::Web::HealthServer} when `config.health_port` is set — that path
|
|
20
|
+
# synthesizes a minimal Rack env and calls this same #call, so the routing
|
|
21
|
+
# and verdict logic live in exactly one place.
|
|
22
|
+
#
|
|
23
|
+
# Routing (env-only, no Rack::Request so it works under the bare-socket
|
|
24
|
+
# HealthServer too):
|
|
25
|
+
# GET /livez → 200 text/plain "ok", unconditionally, NO database access
|
|
26
|
+
# (the serving process is up — that is all liveness means).
|
|
27
|
+
# GET /readyz → build a DataSource, run MCP::HealthAnalyzer#verdict:
|
|
28
|
+
# OK / DEGRADED → 200, STALLED → 503, body = verdict JSON.
|
|
29
|
+
# Any StandardError (DB unreachable) → 503 {"status":"ERROR"},
|
|
30
|
+
# logged via Pgbus.logger (never swallowed).
|
|
31
|
+
# unknown path → 404; non-GET on a known path → 405.
|
|
32
|
+
class HealthApp
|
|
33
|
+
LIVEZ = "/livez"
|
|
34
|
+
READYZ = "/readyz"
|
|
35
|
+
KNOWN_PATHS = [LIVEZ, READYZ].freeze
|
|
36
|
+
private_constant :KNOWN_PATHS
|
|
37
|
+
|
|
38
|
+
TEXT = { "Content-Type" => "text/plain" }.freeze
|
|
39
|
+
JSON_HEADERS = { "Content-Type" => "application/json" }.freeze
|
|
40
|
+
private_constant :TEXT, :JSON_HEADERS
|
|
41
|
+
|
|
42
|
+
# DEGRADED is deliberately "ready": the serving process can still answer,
|
|
43
|
+
# and flapping a pod out of rotation on a transient DEGRADED (a stale
|
|
44
|
+
# sibling, a paused queue) would remove capacity for no gain. Only STALLED
|
|
45
|
+
# — the silent-wedge signal — fails readiness.
|
|
46
|
+
READY_STATUSES = %w[OK DEGRADED].freeze
|
|
47
|
+
private_constant :READY_STATUSES
|
|
48
|
+
|
|
49
|
+
# @param data_source [Pgbus::Web::DataSource, nil] read layer for /readyz.
|
|
50
|
+
# nil (the default) builds a fresh DataSource per readiness check, which
|
|
51
|
+
# avoids serving stale metrics from a long-lived app's memoized instance.
|
|
52
|
+
def initialize(data_source: nil)
|
|
53
|
+
@data_source = data_source
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def call(env)
|
|
57
|
+
path = env["PATH_INFO"].to_s
|
|
58
|
+
return not_found unless KNOWN_PATHS.include?(path)
|
|
59
|
+
return method_not_allowed unless env["REQUEST_METHOD"] == "GET"
|
|
60
|
+
|
|
61
|
+
path == LIVEZ ? livez : readyz
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
private
|
|
65
|
+
|
|
66
|
+
# Liveness: the process is running. No DB, no analyzer, no config lookups.
|
|
67
|
+
def livez
|
|
68
|
+
[200, TEXT.dup, ["ok"]]
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def readyz
|
|
72
|
+
# HealthAnalyzer lives in the MCP namespace, which is excluded from
|
|
73
|
+
# Zeitwerk (its *tools* subclass the optional `mcp` gem). The analyzer
|
|
74
|
+
# itself has no gem dependency, so require just that one file — the
|
|
75
|
+
# standalone health server must work without the `mcp` gem installed.
|
|
76
|
+
require "pgbus/mcp/health_analyzer" unless defined?(Pgbus::MCP::HealthAnalyzer)
|
|
77
|
+
|
|
78
|
+
verdict = Pgbus::MCP::HealthAnalyzer.new(data_source_for_check).verdict
|
|
79
|
+
status = READY_STATUSES.include?(verdict[:status].to_s) ? 200 : 503
|
|
80
|
+
[status, JSON_HEADERS.dup, [verdict.to_json]]
|
|
81
|
+
rescue StandardError => e
|
|
82
|
+
Pgbus.logger.error { "[Pgbus::Web::HealthApp] readiness check failed: #{e.class}: #{e.message}" }
|
|
83
|
+
[503, JSON_HEADERS.dup, [{ status: "ERROR", error: e.message }.to_json]]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Reuse an injected DataSource (tests, an app that wants one shared
|
|
87
|
+
# instance); otherwise build a fresh one each check so per-instance
|
|
88
|
+
# memoization can never serve stale queue/process metrics.
|
|
89
|
+
def data_source_for_check
|
|
90
|
+
@data_source || DataSource.new
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def not_found
|
|
94
|
+
[404, TEXT.dup, ["pgbus: not found"]]
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def method_not_allowed
|
|
98
|
+
[405, TEXT.merge("Allow" => "GET"), ["pgbus: method not allowed"]]
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "socket"
|
|
4
|
+
|
|
5
|
+
module Pgbus
|
|
6
|
+
module Web
|
|
7
|
+
# Standalone HTTP health server for the supervisor process. When
|
|
8
|
+
# `config.health_port` is set, Supervisor#run starts one of these so a
|
|
9
|
+
# Kubernetes kubelet can probe the supervisor directly — the process that
|
|
10
|
+
# forks and watches workers — without the supervisor having to boot Rails,
|
|
11
|
+
# Puma, or the dashboard.
|
|
12
|
+
#
|
|
13
|
+
# It is intentionally tiny: one accept-loop thread over a TCPServer that
|
|
14
|
+
# reads only the HTTP request line (`GET /readyz HTTP/1.1`), synthesizes a
|
|
15
|
+
# minimal Rack env, and hands it to {HealthApp#call}. All routing and the
|
|
16
|
+
# OK/DEGRADED/STALLED → status mapping live in HealthApp, so the mounted-in-
|
|
17
|
+
# Rails path and this standalone path share exactly one implementation.
|
|
18
|
+
#
|
|
19
|
+
# This is a health probe surface, not a general HTTP server: it speaks just
|
|
20
|
+
# enough HTTP/1.0 to answer a kubelet. It only parses the request line
|
|
21
|
+
# (method + path), ignores headers and body, and closes the connection
|
|
22
|
+
# after each response (no keep-alive).
|
|
23
|
+
class HealthServer
|
|
24
|
+
# Cap the request line so a client can't stream unbounded data at the
|
|
25
|
+
# probe port. A real probe request line is well under 100 bytes.
|
|
26
|
+
MAX_REQUEST_LINE = 8_192
|
|
27
|
+
private_constant :MAX_REQUEST_LINE
|
|
28
|
+
|
|
29
|
+
STATUS_TEXT = {
|
|
30
|
+
200 => "OK", 404 => "Not Found", 405 => "Method Not Allowed", 503 => "Service Unavailable"
|
|
31
|
+
}.freeze
|
|
32
|
+
private_constant :STATUS_TEXT
|
|
33
|
+
|
|
34
|
+
# @return [Integer, nil] the bound port. Equals the configured port, or
|
|
35
|
+
# the OS-assigned port when the configured port was 0 (used in tests).
|
|
36
|
+
# nil before #start.
|
|
37
|
+
attr_reader :port
|
|
38
|
+
|
|
39
|
+
def initialize(port:, bind: "127.0.0.1", app: nil, logger: nil)
|
|
40
|
+
@configured_port = port
|
|
41
|
+
@bind = bind
|
|
42
|
+
@app = app || HealthApp.new
|
|
43
|
+
@logger = logger
|
|
44
|
+
@server = nil
|
|
45
|
+
@thread = nil
|
|
46
|
+
@port = nil
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Bind the port and start the accept loop. Idempotent: a second call
|
|
50
|
+
# while running is a no-op (so a supervisor restart path can't double-bind).
|
|
51
|
+
def start
|
|
52
|
+
return if @server
|
|
53
|
+
|
|
54
|
+
@server = TCPServer.new(@bind, @configured_port)
|
|
55
|
+
@port = @server.addr[1]
|
|
56
|
+
@thread = Thread.new { accept_loop }
|
|
57
|
+
logger.info { "[Pgbus::Web::HealthServer] listening on #{@bind}:#{@port} (/livez, /readyz)" }
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Close the listening socket (unblocking the accept loop, which then
|
|
61
|
+
# exits) and join the thread. Safe to call when never started or twice.
|
|
62
|
+
def stop
|
|
63
|
+
server = @server
|
|
64
|
+
@server = nil
|
|
65
|
+
close_socket(server)
|
|
66
|
+
@thread&.join(2)
|
|
67
|
+
@thread = nil
|
|
68
|
+
@port = nil
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
# accept blocks until stop closes the socket, which raises here and ends
|
|
74
|
+
# the loop. Any per-connection error is logged and swallowed so one bad
|
|
75
|
+
# client can never take the probe surface down.
|
|
76
|
+
def accept_loop
|
|
77
|
+
loop do
|
|
78
|
+
server = @server
|
|
79
|
+
break unless server
|
|
80
|
+
|
|
81
|
+
client =
|
|
82
|
+
begin
|
|
83
|
+
server.accept
|
|
84
|
+
rescue IOError, Errno::EBADF, Errno::EINVAL
|
|
85
|
+
break # socket closed by #stop
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
handle_client(client)
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def handle_client(client)
|
|
93
|
+
method, path = read_request_line(client)
|
|
94
|
+
return unless method
|
|
95
|
+
|
|
96
|
+
status, headers, body = @app.call(rack_env(method, path))
|
|
97
|
+
write_response(client, status, headers, body)
|
|
98
|
+
rescue StandardError => e
|
|
99
|
+
logger.warn { "[Pgbus::Web::HealthServer] connection error: #{e.class}: #{e.message}" }
|
|
100
|
+
ensure
|
|
101
|
+
close_socket(client)
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Read and parse only the first line: "METHOD PATH HTTP/x.y". Returns
|
|
105
|
+
# [method, path] or [nil, nil] when the line is missing/garbage so the
|
|
106
|
+
# caller drops the connection without dispatching.
|
|
107
|
+
def read_request_line(client)
|
|
108
|
+
line = client.gets("\r\n", MAX_REQUEST_LINE)
|
|
109
|
+
return [nil, nil] if line.nil?
|
|
110
|
+
|
|
111
|
+
method, path, = line.strip.split(" ", 3)
|
|
112
|
+
return [nil, nil] if method.nil? || path.nil?
|
|
113
|
+
|
|
114
|
+
[method, path]
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Minimal Rack env — HealthApp reads only REQUEST_METHOD and PATH_INFO.
|
|
118
|
+
# Strip any query string from the path so PATH_INFO stays a bare path.
|
|
119
|
+
def rack_env(method, path)
|
|
120
|
+
{ "REQUEST_METHOD" => method, "PATH_INFO" => path.split("?", 2).first }
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def write_response(client, status, headers, body)
|
|
124
|
+
reason = STATUS_TEXT.fetch(status, "OK")
|
|
125
|
+
payload = Array(body).join
|
|
126
|
+
lines = ["HTTP/1.0 #{status} #{reason}"]
|
|
127
|
+
headers.each { |k, v| lines << "#{k}: #{v}" }
|
|
128
|
+
lines << "Content-Length: #{payload.bytesize}"
|
|
129
|
+
lines << "Connection: close"
|
|
130
|
+
client.write("#{lines.join("\r\n")}\r\n\r\n#{payload}")
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def close_socket(socket)
|
|
134
|
+
socket&.close
|
|
135
|
+
rescue IOError, Errno::EBADF
|
|
136
|
+
nil
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def logger
|
|
140
|
+
@logger || Pgbus.logger
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
end
|
|
@@ -16,7 +16,7 @@ module Pgbus
|
|
|
16
16
|
FILTERED = "[FILTERED]"
|
|
17
17
|
|
|
18
18
|
def filter(value)
|
|
19
|
-
return value unless Pgbus.configuration.
|
|
19
|
+
return value unless Pgbus.configuration.web_filter_sensitive
|
|
20
20
|
|
|
21
21
|
case value
|
|
22
22
|
when Hash
|
|
@@ -29,7 +29,7 @@ module Pgbus
|
|
|
29
29
|
end
|
|
30
30
|
|
|
31
31
|
def filter_json(value)
|
|
32
|
-
return value unless Pgbus.configuration.
|
|
32
|
+
return value unless Pgbus.configuration.web_filter_sensitive
|
|
33
33
|
|
|
34
34
|
case value
|
|
35
35
|
when nil then nil
|
|
@@ -46,7 +46,7 @@ module Pgbus
|
|
|
46
46
|
end
|
|
47
47
|
|
|
48
48
|
def parameter_filter
|
|
49
|
-
patterns = Pgbus.configuration.
|
|
49
|
+
patterns = Pgbus.configuration.web_filter_parameters ||
|
|
50
50
|
rails_filter_parameters ||
|
|
51
51
|
DEFAULT_FILTER_PATTERNS
|
|
52
52
|
ActiveSupport::ParameterFilter.new(patterns)
|
|
@@ -28,7 +28,8 @@ module Pgbus
|
|
|
28
28
|
pg_connection: nil,
|
|
29
29
|
logger: Pgbus.logger,
|
|
30
30
|
registry: nil,
|
|
31
|
-
dispatch_queue: nil
|
|
31
|
+
dispatch_queue: nil,
|
|
32
|
+
connection_factory: nil
|
|
32
33
|
)
|
|
33
34
|
@client = client
|
|
34
35
|
@config = config
|
|
@@ -42,7 +43,14 @@ module Pgbus
|
|
|
42
43
|
pg_connection: @pg_connection,
|
|
43
44
|
dispatch_queue: @dispatch_queue,
|
|
44
45
|
health_check_ms: @config.streams_listen_health_check_ms,
|
|
45
|
-
logger: @logger
|
|
46
|
+
logger: @logger,
|
|
47
|
+
# On reconnect the Listener rebuilds its OWN connection via this
|
|
48
|
+
# factory (fresh connect re-resolves DNS, converges on the promoted
|
|
49
|
+
# primary after a failover) instead of resetting a possibly-dead
|
|
50
|
+
# socket. Always provided — even when an initial pg_connection: is
|
|
51
|
+
# injected, the reconnect path builds a fresh raw connection. A test
|
|
52
|
+
# can inject its own factory to avoid touching real configuration.
|
|
53
|
+
connection_factory: connection_factory || -> { build_raw_pg_connection }
|
|
46
54
|
)
|
|
47
55
|
@dispatcher = StreamEventDispatcher.new(
|
|
48
56
|
client: @client,
|
|
@@ -143,7 +151,20 @@ module Pgbus
|
|
|
143
151
|
end
|
|
144
152
|
end
|
|
145
153
|
|
|
154
|
+
# Build the INITIAL LISTEN connection: raw connect, reject a replica
|
|
155
|
+
# fatally (a misconfiguration at boot), then run the one-shot delivery
|
|
156
|
+
# self-probe. The reconnect path uses build_raw_pg_connection directly
|
|
157
|
+
# (validation retries there, probe is skipped) — see the factory wired
|
|
158
|
+
# into Listener.new above.
|
|
146
159
|
def build_pg_connection
|
|
160
|
+
probe(validate_primary(build_raw_pg_connection))
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Raw PG.connect with no validation or probe. This is what the Listener's
|
|
164
|
+
# connection_factory calls on every reconnect attempt: the reconnect loop
|
|
165
|
+
# runs its own PrimaryValidator (retrying on a replica) and skips the
|
|
166
|
+
# probe to stay cheap, mirroring Pgbus::Process::NotifyListener.
|
|
167
|
+
def build_raw_pg_connection
|
|
147
168
|
require "pg" unless defined?(::PG::Connection)
|
|
148
169
|
opts = @config.streams_connection_options
|
|
149
170
|
case opts
|
|
@@ -167,6 +188,41 @@ module Pgbus
|
|
|
167
188
|
"Cannot build streamer PG connection from #{opts.class}"
|
|
168
189
|
end
|
|
169
190
|
end
|
|
191
|
+
|
|
192
|
+
# Reject a connection that landed on a read-only replica before the
|
|
193
|
+
# streamer wires it into a Listener. After a failover, stale DNS can
|
|
194
|
+
# point this fresh PG.connect at the demoted master; NOTIFY fires only
|
|
195
|
+
# on the primary, so the streamer would sit deaf. Unlike the reconnect
|
|
196
|
+
# loop (which retries), this runs once at Instance construction (Puma
|
|
197
|
+
# worker boot), so a replica here is a fatal misconfiguration: re-raise
|
|
198
|
+
# as a ConfigurationError naming the direct-connection overrides.
|
|
199
|
+
def validate_primary(pg_connection)
|
|
200
|
+
Pgbus::Process::PrimaryValidator.validate_primary!(pg_connection)
|
|
201
|
+
rescue Pgbus::Process::ReplicaConnectionError => e
|
|
202
|
+
close_quietly(pg_connection)
|
|
203
|
+
raise Pgbus::ConfigurationError,
|
|
204
|
+
"Streamer LISTEN connection landed on a read-only replica " \
|
|
205
|
+
"(#{e.message}). NOTIFY fires only on the primary, so the " \
|
|
206
|
+
"streamer would never wake. Point the streamer at a DIRECT " \
|
|
207
|
+
"primary via streams_database_url (or streams_host / " \
|
|
208
|
+
"streams_port)."
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def close_quietly(pg_connection)
|
|
212
|
+
pg_connection.close if pg_connection.respond_to?(:close)
|
|
213
|
+
rescue StandardError
|
|
214
|
+
nil
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
# One-shot LISTEN/NOTIFY delivery self-probe on the freshly built
|
|
218
|
+
# connection. A transaction-mode pooler or replica that silently breaks
|
|
219
|
+
# NOTIFY is surfaced with an actionable error (naming the streams_*
|
|
220
|
+
# overrides); the streamer still starts and degrades to slow SSE
|
|
221
|
+
# updates rather than crashing. Returns the connection unchanged.
|
|
222
|
+
def probe(pg_connection)
|
|
223
|
+
Pgbus::Process::NotifyProbe.probe_notify_delivery!(pg_connection, logger: @logger)
|
|
224
|
+
pg_connection
|
|
225
|
+
end
|
|
170
226
|
end
|
|
171
227
|
end
|
|
172
228
|
end
|
|
@@ -19,10 +19,11 @@ module Pgbus
|
|
|
19
19
|
#
|
|
20
20
|
# Health check: `wait_for_notify(timeout)` returns nil on timeout. When
|
|
21
21
|
# it does, the listener runs `SELECT 1` as a TCP keepalive. If that
|
|
22
|
-
# raises, the
|
|
23
|
-
#
|
|
24
|
-
# (silently dropped LISTEN connections from NAT / PG
|
|
25
|
-
# blips)
|
|
22
|
+
# raises, the listener rebuilds a FRESH connection via connection_factory
|
|
23
|
+
# and re-LISTENs every channel in `@listening_to`. This is the fix for
|
|
24
|
+
# design doc §11 #1 (silently dropped LISTEN connections from NAT / PG
|
|
25
|
+
# restart / network blips) — a fresh connect also re-resolves DNS and
|
|
26
|
+
# converges on the promoted primary after a failover.
|
|
26
27
|
#
|
|
27
28
|
# NOTIFY channel naming (from pgmq_v1.11.0.sql:1634):
|
|
28
29
|
# PG_NOTIFY('pgmq.' || TG_TABLE_NAME || '.' || TG_OP, NULL)
|
|
@@ -38,13 +39,24 @@ module Pgbus
|
|
|
38
39
|
CHANNEL_PREFIX = "pgmq.q_"
|
|
39
40
|
CHANNEL_SUFFIX = ".INSERT"
|
|
40
41
|
|
|
42
|
+
RECONNECT_BACKOFF_SECONDS = 0.5
|
|
43
|
+
|
|
41
44
|
attr_reader :listening_to
|
|
42
45
|
|
|
43
|
-
|
|
46
|
+
# @param connection_factory [#call] builds a FRESH PG connection on each
|
|
47
|
+
# reconnect attempt (the Instance passes -> { build_raw_pg_connection }).
|
|
48
|
+
# Required — the reconnect loop always rebuilds rather than resetting a
|
|
49
|
+
# possibly-dead socket, so every caller (production and tests) must pass
|
|
50
|
+
# one.
|
|
51
|
+
def initialize(pg_connection:, dispatch_queue:, health_check_ms:,
|
|
52
|
+
connection_factory:, logger: Pgbus.logger)
|
|
53
|
+
raise ArgumentError, "connection_factory is required" unless connection_factory.respond_to?(:call)
|
|
54
|
+
|
|
44
55
|
@conn = pg_connection
|
|
45
56
|
@dispatch_queue = dispatch_queue
|
|
46
57
|
@health_check_ms = health_check_ms
|
|
47
58
|
@logger = logger
|
|
59
|
+
@connection_factory = connection_factory
|
|
48
60
|
@listening_to = Set.new
|
|
49
61
|
@commands = Queue.new
|
|
50
62
|
@running = false
|
|
@@ -197,28 +209,64 @@ module Pgbus
|
|
|
197
209
|
@conn.exec("SELECT 1")
|
|
198
210
|
end
|
|
199
211
|
|
|
212
|
+
# Retry reconnect until we succeed (fresh conn + every channel
|
|
213
|
+
# re-LISTENed) or #stop flips @running to false. Mirrors
|
|
214
|
+
# NotifyListener#reconnect!: a single failed attempt never returns with
|
|
215
|
+
# a dead connection while running, so ensure_listening acks can't
|
|
216
|
+
# succeed vacuously against a broken LISTEN socket and strand SSE
|
|
217
|
+
# clients.
|
|
200
218
|
def reconnect!
|
|
201
|
-
|
|
202
|
-
#
|
|
203
|
-
#
|
|
204
|
-
#
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
@conn
|
|
211
|
-
|
|
219
|
+
# Snapshot the canonical subscription set BEFORE tearing down the old
|
|
220
|
+
# connection. We rebuild @listening_to from this and only publish it
|
|
221
|
+
# once every channel has been re-LISTENed on the new conn, so a
|
|
222
|
+
# mid-loop LISTEN failure never loses channels.
|
|
223
|
+
channels = @listening_to.to_a
|
|
224
|
+
loop do
|
|
225
|
+
return unless @running
|
|
226
|
+
|
|
227
|
+
close_quietly(@conn)
|
|
228
|
+
@conn = nil
|
|
229
|
+
new_conn = nil
|
|
230
|
+
begin
|
|
231
|
+
new_conn = @connection_factory.call
|
|
232
|
+
# Reject a replica before re-LISTENing. A fresh connect re-resolves
|
|
233
|
+
# DNS, so backing off and retrying converges on the promoted primary
|
|
234
|
+
# once DNS catches up after a failover. NOTIFY fires only on the
|
|
235
|
+
# primary; re-LISTENing on a replica registers channels that never wake.
|
|
236
|
+
Pgbus::Process::PrimaryValidator.validate_primary!(new_conn)
|
|
237
|
+
channels.each { |channel| new_conn.exec(%(LISTEN "#{channel}")) }
|
|
238
|
+
rescue PG::Error, Pgbus::Process::ReplicaConnectionError, Pgbus::ConfigurationError => e
|
|
239
|
+
# The factory may raise a ConfigurationError (a bad
|
|
240
|
+
# streams_connection_options, e.g. a pooled AR connection) or return
|
|
241
|
+
# a live conn before a later LISTEN raised (or validate_primary!
|
|
242
|
+
# rejected a replica). Close any partial conn so repeated failures
|
|
243
|
+
# don't leak PG connections, then back off and retry rather than
|
|
244
|
+
# letting the exception kill the listener thread silently.
|
|
245
|
+
close_quietly(new_conn)
|
|
246
|
+
@logger.error { "[Pgbus::Streamer::Listener] reconnect failed: #{e.class}: #{e.message}" }
|
|
247
|
+
sleep RECONNECT_BACKOFF_SECONDS
|
|
248
|
+
next
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
@conn = new_conn
|
|
252
|
+
@listening_to = Set.new(channels)
|
|
253
|
+
return
|
|
212
254
|
end
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
# Close a PG connection we're discarding (the old conn at the top of a
|
|
258
|
+
# reconnect cycle, or a half-built one whose LISTEN raised). Best-effort.
|
|
259
|
+
def close_quietly(conn)
|
|
260
|
+
conn&.close if conn.respond_to?(:close)
|
|
261
|
+
rescue StandardError
|
|
262
|
+
nil
|
|
217
263
|
end
|
|
218
264
|
|
|
219
265
|
def safe_unlisten_all
|
|
220
266
|
@listening_to.each do |channel|
|
|
221
|
-
@conn
|
|
267
|
+
# @conn may be nil if #stop interrupted the reconnect loop between
|
|
268
|
+
# closing the old connection and publishing a new one.
|
|
269
|
+
@conn&.exec(%(UNLISTEN "#{channel}"))
|
|
222
270
|
rescue PG::Error
|
|
223
271
|
# connection may be dead; nothing we can do
|
|
224
272
|
end
|
data/lib/pgbus.rb
CHANGED
|
@@ -10,6 +10,24 @@ module Pgbus
|
|
|
10
10
|
# this to be configurable.
|
|
11
11
|
DEAD_LETTER_SUFFIX = "_dlq"
|
|
12
12
|
|
|
13
|
+
# Error-hierarchy policy (the 1.0 contract, issue #282):
|
|
14
|
+
#
|
|
15
|
+
# * OPERATIONAL errors — a pgbus subsystem failed to do its job at runtime
|
|
16
|
+
# (bad configuration, a pool that's shutting down, a batch enqueue that
|
|
17
|
+
# came back short, a rejected GlobalID, a missing PGMQ version) — descend
|
|
18
|
+
# from Pgbus::Error. `rescue Pgbus::Error` is meant to catch every one of
|
|
19
|
+
# them; that is the whole point of the hierarchy.
|
|
20
|
+
#
|
|
21
|
+
# * ARGUMENT-SHAPE errors — the caller handed a method a malformed argument
|
|
22
|
+
# (a stream name too long for the queue-name budget, an unparseable
|
|
23
|
+
# cursor, a capsule DSL line that doesn't parse) — stay ArgumentError
|
|
24
|
+
# subclasses. That's what ArgumentError means, and rescuing it is the
|
|
25
|
+
# caller's responsibility, not an operational concern. These live in
|
|
26
|
+
# their own files: Streams::StreamNameTooLong, Streams::Cursor::InvalidCursor,
|
|
27
|
+
# Configuration::CapsuleDSL::ParseError.
|
|
28
|
+
#
|
|
29
|
+
# When adding a new error class, decide which half it belongs to and parent
|
|
30
|
+
# it accordingly — do not default to StandardError.
|
|
13
31
|
class Error < StandardError; end
|
|
14
32
|
class ConfigurationError < Error; end
|
|
15
33
|
class SerializationError < Error; end
|
|
@@ -20,6 +38,41 @@ module Pgbus
|
|
|
20
38
|
class SchemaNotReady < Error; end
|
|
21
39
|
class ReadTimeoutError < Error; end
|
|
22
40
|
|
|
41
|
+
# Raised by the ActiveJob adapter when a batch enqueue (perform_all_later)
|
|
42
|
+
# comes back with a msg_id count that doesn't match the number of jobs sent —
|
|
43
|
+
# a data-integrity signal that some jobs may not have been persisted.
|
|
44
|
+
class EnqueueError < Error; end
|
|
45
|
+
|
|
46
|
+
# Raised by the execution pools when work can't be accepted: the pool is
|
|
47
|
+
# shutting down, or it's momentarily at capacity. Consumer-reachable during
|
|
48
|
+
# job submission, so it descends from Pgbus::Error rather than RuntimeError.
|
|
49
|
+
class ExecutionPoolError < Error; end
|
|
50
|
+
|
|
51
|
+
# Raised by Client read paths when the in-memory connection-health circuit
|
|
52
|
+
# breaker (Client::ConnectionHealth) is open: the database has failed enough
|
|
53
|
+
# consecutive connection attempts that reads now fail fast without a pool
|
|
54
|
+
# checkout, sparing a dead database from the whole fleet re-polling and the
|
|
55
|
+
# error tracker from per-poll noise. Workers rescue this and idle until the
|
|
56
|
+
# breaker's backoff window admits a probe. See issue #197.
|
|
57
|
+
class ConnectionCircuitOpenError < Error; end
|
|
58
|
+
|
|
59
|
+
module Process
|
|
60
|
+
# A LISTEN connection landed on a read-only replica (pg_is_in_recovery() =>
|
|
61
|
+
# t). After a failover, stale DNS can point a fresh connection at the
|
|
62
|
+
# demoted master; NOTIFY fires only on the primary, so such a connection
|
|
63
|
+
# connects "successfully" but never wakes. Raised by
|
|
64
|
+
# Process::PrimaryValidator so the listener rejects it and retries with
|
|
65
|
+
# backoff (re-resolving DNS each attempt).
|
|
66
|
+
#
|
|
67
|
+
# Defined here rather than in process/primary_validator.rb because Zeitwerk
|
|
68
|
+
# requires each managed file to define exactly the constant matching its
|
|
69
|
+
# path — a second class in that file would break eager_load. lib/pgbus.rb
|
|
70
|
+
# is the (require-loaded) gem entry point, so it can define the error and
|
|
71
|
+
# explicitly namespace Process without conflicting with Zeitwerk's
|
|
72
|
+
# management of the lib/pgbus/process/ directory.
|
|
73
|
+
class ReplicaConnectionError < Error; end
|
|
74
|
+
end
|
|
75
|
+
|
|
23
76
|
class << self
|
|
24
77
|
# Process-global flag set by Worker#graceful_shutdown so the adapter
|
|
25
78
|
# can report stopping? to ActiveJob::Continuation (Rails 8.1+).
|
|
@@ -35,6 +88,7 @@ module Pgbus
|
|
|
35
88
|
loader.inflector.inflect(
|
|
36
89
|
"pgbus" => "Pgbus",
|
|
37
90
|
"cli" => "CLI",
|
|
91
|
+
"dlq" => "DLQ",
|
|
38
92
|
"dsl" => "DSL",
|
|
39
93
|
"capsule_dsl" => "CapsuleDSL",
|
|
40
94
|
"mcp" => "MCP"
|
|
@@ -92,6 +146,11 @@ module Pgbus
|
|
|
92
146
|
|
|
93
147
|
def configure
|
|
94
148
|
yield configuration
|
|
149
|
+
# Fail loud at boot on an invalid value rather than leaving it dormant
|
|
150
|
+
# until a worker path consumes it. Checking the flag after the yield
|
|
151
|
+
# lets a block opt out for itself via `c.eager_validation = false`.
|
|
152
|
+
configuration.validate! if configuration.eager_validation
|
|
153
|
+
configuration
|
|
95
154
|
end
|
|
96
155
|
|
|
97
156
|
def client
|
|
@@ -150,6 +209,24 @@ module Pgbus
|
|
|
150
209
|
Streams::Key.stream_key!(key)
|
|
151
210
|
end
|
|
152
211
|
|
|
212
|
+
# Publish an event to the bus — the top-level shortcut for
|
|
213
|
+
# `Pgbus::EventBus::Publisher.publish`, symmetric with `Pgbus.stream`.
|
|
214
|
+
#
|
|
215
|
+
# Pgbus.publish("orders.created", { order_id: 42 })
|
|
216
|
+
# Pgbus.publish("orders.created", order, headers: { "x-trace" => id }, delay: 30)
|
|
217
|
+
def publish(routing_key, payload, headers: nil, delay: 0)
|
|
218
|
+
EventBus::Publisher.publish(routing_key, payload, headers: headers, delay: delay)
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# Publish an event with a delay. Mirrors
|
|
222
|
+
# `Pgbus::EventBus::Publisher.publish_later`; `delay:` is required (a
|
|
223
|
+
# publish_later with no delay is just publish).
|
|
224
|
+
#
|
|
225
|
+
# Pgbus.publish_later("orders.reminder", { order_id: 42 }, delay: 1.hour)
|
|
226
|
+
def publish_later(routing_key, payload, delay:, headers: nil)
|
|
227
|
+
EventBus::Publisher.publish_later(routing_key, payload, delay: delay, headers: headers)
|
|
228
|
+
end
|
|
229
|
+
|
|
153
230
|
def reset!
|
|
154
231
|
@client&.close
|
|
155
232
|
@client = nil
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
namespace :pgbus do
|
|
4
|
+
desc "Run environment diagnostics (config, DB, PGMQ, queues, LISTEN/NOTIFY, process liveness)"
|
|
5
|
+
task doctor: :environment do
|
|
6
|
+
require "pgbus/doctor"
|
|
7
|
+
|
|
8
|
+
doctor = Pgbus::Doctor.new
|
|
9
|
+
puts doctor.report
|
|
10
|
+
exit 1 unless doctor.success?
|
|
11
|
+
end
|
|
12
|
+
end
|