pgbus 0.21.2 → 0.21.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +30 -0
- data/README.md +3 -2
- data/lib/pgbus/configuration.rb +27 -0
- data/lib/pgbus/event.rb +10 -0
- data/lib/pgbus/mcp/health_analyzer.rb +40 -4
- data/lib/pgbus/mcp/tools/health_tool.rb +4 -2
- data/lib/pgbus/pgmq_schema/fixups/pgmq_v1.13.1.sql +265 -0
- data/lib/pgbus/pgmq_schema/pgmq_v1.13.1.sql +2324 -0
- data/lib/pgbus/pgmq_schema.rb +34 -4
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source.rb +8 -3
- data/sig/pgbus/configuration.rbs +2 -0
- metadata +3 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 34cf3117ba1bd7fdc9355f8593496af02af9e7e878efc5247856ba2afd7533e8
|
|
4
|
+
data.tar.gz: b301f2fcf44e83d9438ece6df8f7e94bb627ea91b83070e8e56ce6e3a10f66ae
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 95ccb84dcc956d3d4e483976ee7e62f8fb4525622761e9f767c3cf9237b9de7069c36ab5aba05820a0acbe3da564ca105add4512b010e085dcb11a51ffb97fd8
|
|
7
|
+
data.tar.gz: cf6fe7554b273ad00988f449a18410ba456520f808337a316671302fcef507e16037e8a64764bb08130521337be065fdc6b9939a7ea788afa872db0423f9775a
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,31 @@
|
|
|
1
1
|
## [Unreleased]
|
|
2
2
|
|
|
3
|
+
## [0.21.4] - 2026-10-28
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
|
|
7
|
+
- **`Pgbus::Event#dig` reads a nested payload value, the nested counterpart of `Event#[]` (issue #594).** `event.dig("meta", "type")` reads Symbol keys as Strings at every level (payloads are JSON-decoded) and passes Integer indexes through (`event.dig("data", "items", 0, "id")`); a missing key or a payload that is not a Hash returns `nil`, as `#[]` does, and an intermediate value without `#dig` raises `TypeError`, as `Hash#dig` does. Refs #594.
|
|
8
|
+
|
|
9
|
+
### Changed
|
|
10
|
+
|
|
11
|
+
- **PGMQ: vendored schema v1.13.1, a security release upstream asks everyone on 1.13.0 or earlier to take (issue #586).** Every PGMQ function pins `search_path = pg_catalog, pg_temp` and schema-qualifies its built-ins, so a role that can create objects on another role's search path can no longer shadow a built-in and run code with that role's privileges. `pg_monitor` loses `SELECT` on every table and sequence in the `pgmq` schema: it keeps `SELECT` on `pgmq.meta`, `SELECT (vt, enqueued_at)` on each queue table and `SELECT` on each queue's msg_id sequence, what `pgmq.metrics()`, `pgmq.metrics_all()` and `pgmq.list_queues()` need; a monitoring role that read message payloads through `pg_monitor` no longer can. Fresh embedded installs get 1.13.1. Existing *embedded* installs: `rake pgbus:pgmq:status`, then `rails generate pgbus:upgrade_pgmq` + `rails db:migrate` (`rails db:migrate:pgbus` on a separate-database install). Its new fixup (`lib/pgbus/pgmq_schema/fixups/pgmq_v1.13.1.sql`, upstream's migration blocks) revokes `pg_monitor`'s default privileges and whole-table grants in `pgmq`, grants the metrics columns and sequence on every existing queue, and rebinds a PGMQ column default that resolved to `now()` or `to_timestamp()` outside `pg_catalog`; a grant the migrating role cannot revoke raises a `WARNING` starting `pgmq:` instead of failing (Rails logs it only with `config.active_record.db_warnings_action` set; otherwise it goes to stderr), and a default it cannot alter stops the migration. Upstream's default rebinding selects extension member tables only, which would skip every embedded install, so the fixup selects pgmq's tables by name. `Pgbus::PgmqSchema.install_sql` now restores the caller's `search_path` after the vendored SQL, because 1.13.1 pins it with a transaction-local setting that would otherwise outlive the install inside a migration and send the next unqualified `CREATE TABLE` to `pg_catalog`. An `:extension` install gets 1.13.1 from `ALTER EXTENSION pgmq UPDATE`. Refs #586.
|
|
12
|
+
|
|
13
|
+
### Fixed
|
|
14
|
+
|
|
15
|
+
- **NOTIFY wakeups fire for a queue whose name has an uppercase letter (issue #586).** PGMQ's insert trigger rebuilt the queue name from its lowercased table name, which matched no `pgmq.notify_insert_throttle` row, so it never sent the NOTIFY and that queue's workers polled. PGMQ 1.13.1 passes the queue name to the trigger, and the upgrade migration's NOTIFY repair re-creates every queue's trigger that way; the channel, `pgmq.q_<queue name>.INSERT`, keeps the queue name's case, which is what pgbus LISTENs on. Refs #586.
|
|
16
|
+
|
|
17
|
+
## [0.21.3] - 2026-10-22
|
|
18
|
+
|
|
19
|
+
### Added
|
|
20
|
+
|
|
21
|
+
- **`config.health_unclaimed_grace` (default 60 seconds) and the per-queue metric `oldest_unread_claimable_age_sec` (issue #589).** The grace is how long a visible, never-read (`read_ct = 0`) message must have been claimable before the health verdict counts it as "never claimed while draining processes are alive". It takes a number of seconds or an `ActiveSupport::Duration`; `0` restores the old rule (any such message counts), and `validate!` raises `ConfigurationError` on nil, a negative or a non-finite value. `Web::DataSource#queues_with_metrics` and `#queue_detail` report `oldest_unread_claimable_age_sec`, the seconds since the oldest visible `read_ct = 0` message became claimable (`min(vt)`, so a delayed job's age starts when its wait expires and a retried message is left out), nil when there is none. The verdict's `summary` carries it as the oldest over the drained queues, and the STALLED reason names it (`never claimed, oldest claimable for 74s`). `Pgbus::MCP::HealthAnalyzer.new` takes an optional `grace:`. Refs #589.
|
|
22
|
+
|
|
23
|
+
### Fixed
|
|
24
|
+
|
|
25
|
+
- **`/readyz` and `pgbus_health` no longer flap to `STALLED` on a healthy fleet (issue #589).** The verdict reported `STALLED` (and the mounted `/readyz` 503) whenever a drained queue held one visible, never-read message while a worker or consumer was alive, however briefly it had been claimable: every enqueue and every delayed job whose wait just expired is such a message until a worker's next poll, so a busy install answered 503 on a large share of checks. That branch now counts a queue only once its oldest such message has been claimable for `health_unclaimed_grace` seconds; a wedged worker still reads `STALLED` after the grace. A queue hash without the age (an older or stubbed data source) still counts at once. The worker-reported stalled branch (heart-beating, claim loop not advancing) is unchanged. Refs #589.
|
|
26
|
+
|
|
27
|
+
## [0.21.2] - 2026-10-21
|
|
28
|
+
|
|
3
29
|
### Added
|
|
4
30
|
|
|
5
31
|
- **`config.boot_timeout` (default 120 seconds): the supervisor kills a child that never checks in (issue #587).** A fork checks in when it registers its `pgbus_processes` row on this host or writes its first liveness byte. One that has done neither `boot_timeout` seconds after the fork is logged at ERROR, sent `TTIN`, given 2 seconds, then SIGKILLed, and the normal restart path forks it again. A boot-deadline kill always counts as a crash, whatever the child's uptime, so a boot that hangs every time is re-forked with the crash-loop backoff (1 s doubling to 60 s) instead of every `boot_timeout`. Applies to workers, consumers, the dispatcher, the scheduler and the outbox poller. `nil` turns it off; anything else must be a positive number, or `validate!` raises `ConfigurationError`. When the watchdog's read of the process table fails, the boot deadline waits for a pass that can read it. Refs #587.
|
|
@@ -17,6 +43,10 @@
|
|
|
17
43
|
- **A failed job's retry backoff applies under priority routing (issue #582).** With `priority_levels > 1` the executor set the backoff's visibility timeout on the logical queue's base table, which priority routing never creates; the call raised, was logged at debug, and the job came back after the read's visibility timeout instead of its backoff delay. The backoff now goes to the `_pN` sub-queue the worker read the message from, as the visibility heartbeat and the archive already did. Refs #582.
|
|
18
44
|
- **Two priority sub-queues' failures with the same msg_id keep a row each (issue #583).** Each `_pN` sub-queue numbers its messages from its own sequence, and `pgbus_failed_events` was unique on the logical queue and the msg_id, so a second job failing with the same msg_id in another sub-queue overwrote the first one's error and kept its payload: the dashboard showed one failed job where two had failed, Retry and Discard acted on the first job's message and deleted the only row, and the first job's success cleared the second one's failure. Rows now record the physical table (`source_queue`) and are unique on `(queue_name, msg_id, source_queue)`. The executor, the event consumer and the worker's zombie check read, write and clear only the row of the table the message came from. On the Jobs page a failed row joins only its own table's message and shows as an orphan once that message is gone, Retry, Discard and Discard all look the message up in the table the row names, and Discard releases the job's uniqueness lock bounded by when that table's message was sent, not by a sibling sub-queue's. A row written before the migration has no `source_queue` and is treated as before: it can belong to any table of its queue. Refs #583.
|
|
19
45
|
|
|
46
|
+
## [0.21.1] - 2026-10-19
|
|
47
|
+
|
|
48
|
+
### Fixed
|
|
49
|
+
|
|
20
50
|
- **The dashboard's Retry, Discard and Discard all reach a failed job's message under priority routing (issue #577).** A failed-job row records the logical queue only, and with `priority_levels > 1` the message sits in a `_pN` sub-queue, never in the base table. Retry reset the visibility timeout on `pgmq.q_pgbus_<queue>`, which priority routing never creates, and answered "Could not retry job."; Discard and Discard all archived from that same table, so they reported the job discarded while its message stayed queued and ran again. Each sub-queue also numbers its messages from its own sequence, so the msg_id alone could name another job's message in a sibling sub-queue. The dashboard now finds the sub-queue whose message has the row's msg_id and carries the job's `job_id` (`Pgbus::Client#locate_message`, which returns that physical table or nil, and `#locate_messages` for many at once, one query per table), resets or archives the message there, and leaves a same-numbered message elsewhere alone. Discard all locates a queue's messages together and archives them in one batch per table. When no sub-queue holds it, Retry enqueues a fresh copy as before. Without priority routing there is one table and nothing changes, except that a message reusing the msg_id under another `job_id` is no longer reset or archived in the job's place. Refs #577.
|
|
21
51
|
|
|
22
52
|
## [0.21.0] - 2026-10-19
|
data/README.md
CHANGED
|
@@ -1260,7 +1260,7 @@ All tools below are read-only — none mutates state, and there is no raw-SQL pa
|
|
|
1260
1260
|
|
|
1261
1261
|
| Tool | Purpose |
|
|
1262
1262
|
|------|---------|
|
|
1263
|
-
| `pgbus_health` | One-call verdict: `OK` / `DEGRADED` / `STALLED`. STALLED is the silent-worker-wedge signal
|
|
1263
|
+
| `pgbus_health` | One-call verdict: `OK` / `DEGRADED` / `STALLED`. STALLED is the silent-worker-wedge signal: a worker reports its claim loop stalled, or a visible backlog older than `health_unclaimed_grace` (default 60s) while workers heart-beat but don't claim. The summary carries `oldest_unread_claimable_age_sec`. Suitable for automated alerting. |
|
|
1264
1264
|
| `pgbus_queues` | All queues: depth, visible count, oldest-message age, paused state. |
|
|
1265
1265
|
| `pgbus_queue_detail` | Per-queue metrics + paused state + table health (dead tuples, bloat, vacuum age). |
|
|
1266
1266
|
| `pgbus_processes` | Every process with kind, pid, heartbeat age, and `healthy`/`stale`/`stalled` status. |
|
|
@@ -1348,7 +1348,7 @@ For the **HTTP** transport, point the client at the mounted URL with a streamabl
|
|
|
1348
1348
|
|
|
1349
1349
|
For orchestrators like Kubernetes, Pgbus exposes two HTTP probes: `/livez` (is the serving process up?) and `/readyz`. Readiness means different things in the two places the probes are served:
|
|
1350
1350
|
|
|
1351
|
-
- **Mounted in Rails** (`Pgbus::Web::HealthApp`): `/readyz` runs the cluster-wide `OK` / `DEGRADED` / `STALLED` verdict, same as the MCP `pgbus_health` tool — `STALLED` (
|
|
1351
|
+
- **Mounted in Rails** (`Pgbus::Web::HealthApp`): `/readyz` runs the cluster-wide `OK` / `DEGRADED` / `STALLED` verdict, same as the MCP `pgbus_health` tool — `STALLED` (a never-claimed backlog older than `health_unclaimed_grace` while workers heart-beat but don't claim) fails readiness.
|
|
1352
1352
|
- **Standalone from the supervisor** (`health_port`): `/readyz` is **container-local** — did *this* supervisor finish booting, and are all the children *it* forked alive? That is the signal a rolling deploy's health gate needs; the cluster verdict would let a brand-new container pass on the strength of the *old* container's workers.
|
|
1353
1353
|
|
|
1354
1354
|
| Path | Method | 200 | 503 | Touches DB |
|
|
@@ -2560,6 +2560,7 @@ Curated headline options for the README. The full operator reference (with types
|
|
|
2560
2560
|
| `statsd_port` | `8125` | StatsD UDP port (used when `metrics_backend = :statsd`) |
|
|
2561
2561
|
| `health_port` | `nil` | Port for standalone HTTP liveness/readiness probes served by the supervisor; nil disables |
|
|
2562
2562
|
| `health_bind` | `"127.0.0.1"` | Bind address for the standalone health server |
|
|
2563
|
+
| `health_unclaimed_grace` | `60` | Seconds a visible, never-read message must have been claimable before the health verdict reports it as never claimed (`STALLED`); `0` counts any such message |
|
|
2563
2564
|
| `pgmq_schema_mode` | `:auto` | PGMQ schema install mode (`:auto`, `:extension`, `:embedded`) |
|
|
2564
2565
|
| `doctor_on_boot` | `nil` | Run doctor inside the booting supervisor: `nil`/`false` = off, `:report` = log and boot, `:strict` = refuse to boot on a fatal check. Also via `--doctor` / `--doctor-strict`. |
|
|
2565
2566
|
| `eager_validation` | `true` | Run `Configuration#validate!` automatically after the `Pgbus.configure` block yields; an invalid value raises `Pgbus::ConfigurationError` at boot. Set `false` to suppress and validate manually. |
|
data/lib/pgbus/configuration.rb
CHANGED
|
@@ -241,6 +241,14 @@ module Pgbus
|
|
|
241
241
|
# plain TCPServer bound to `health_bind` (default localhost).
|
|
242
242
|
attr_accessor :health_port, :health_bind
|
|
243
243
|
|
|
244
|
+
# Seconds a visible, never-read (read_ct = 0) message must have been
|
|
245
|
+
# claimable before the health verdict counts it as "never claimed while
|
|
246
|
+
# draining processes are alive" (issue #589). Below it a worker's next
|
|
247
|
+
# poll is still coming, so a fresh enqueue or a delayed job whose wait just
|
|
248
|
+
# expired no longer reads STALLED. 0 restores the pre-0.22 rule (any
|
|
249
|
+
# visible read_ct = 0 row counts). Default 60.
|
|
250
|
+
attr_reader :health_unclaimed_grace
|
|
251
|
+
|
|
244
252
|
# Streams (turbo-rails replacement, SSE-based)
|
|
245
253
|
attr_accessor :streams_enabled, :streams_path, :streams_signed_name_secret,
|
|
246
254
|
:streams_default_retention, :streams_retention, :streams_heartbeat_interval, :streams_peer_close_interval,
|
|
@@ -419,6 +427,7 @@ module Pgbus
|
|
|
419
427
|
# HTTP health endpoints — nil port disables the standalone server.
|
|
420
428
|
@health_port = nil
|
|
421
429
|
@health_bind = "127.0.0.1"
|
|
430
|
+
@health_unclaimed_grace = 60
|
|
422
431
|
|
|
423
432
|
@streams_enabled = true
|
|
424
433
|
@streams_path = nil
|
|
@@ -951,6 +960,8 @@ module Pgbus
|
|
|
951
960
|
|
|
952
961
|
raise Pgbus::ConfigurationError, "health_bind must be a non-empty String" unless health_bind.is_a?(String) && !health_bind.empty?
|
|
953
962
|
|
|
963
|
+
validate_health_unclaimed_grace!
|
|
964
|
+
|
|
954
965
|
raise Pgbus::ConfigurationError, "statsd_host must be a non-empty String" unless statsd_host.is_a?(String) && !statsd_host.empty?
|
|
955
966
|
|
|
956
967
|
unless statsd_port.is_a?(Integer) && statsd_port.between?(1, 65_535)
|
|
@@ -1539,6 +1550,14 @@ module Pgbus
|
|
|
1539
1550
|
@recurring_execution_retention = coerce_duration!(value, :recurring_execution_retention)
|
|
1540
1551
|
end
|
|
1541
1552
|
|
|
1553
|
+
# Not coerce_duration!: 0 is valid here (the pre-0.22 rule), and nil is not
|
|
1554
|
+
# a "disabled" sentinel. A Duration becomes integer seconds; anything else
|
|
1555
|
+
# is kept for validate! to judge.
|
|
1556
|
+
def health_unclaimed_grace=(value)
|
|
1557
|
+
duration = defined?(ActiveSupport::Duration) && value.is_a?(ActiveSupport::Duration)
|
|
1558
|
+
@health_unclaimed_grace = duration ? value.to_i : value
|
|
1559
|
+
end
|
|
1560
|
+
|
|
1542
1561
|
# recurring_tasks_file (singular) is deprecated in favor of the plural
|
|
1543
1562
|
# recurring_tasks_files. The engine still writes the singular internally to
|
|
1544
1563
|
# record which default file was loaded, so it stays a plain accessor; the
|
|
@@ -1867,6 +1886,14 @@ module Pgbus
|
|
|
1867
1886
|
raise Pgbus::ConfigurationError, "immediate_shutdown_grace must be a finite number >= 0"
|
|
1868
1887
|
end
|
|
1869
1888
|
|
|
1889
|
+
# A finite real number >= 0; nil is not "off" (0 is).
|
|
1890
|
+
def validate_health_unclaimed_grace!
|
|
1891
|
+
grace = health_unclaimed_grace
|
|
1892
|
+
return if grace.is_a?(Numeric) && grace.real? && grace.finite? && grace >= 0
|
|
1893
|
+
|
|
1894
|
+
raise Pgbus::ConfigurationError, "health_unclaimed_grace must be a finite number of seconds >= 0, got #{grace.inspect}"
|
|
1895
|
+
end
|
|
1896
|
+
|
|
1870
1897
|
def validate_positive_duration!(numeric, name)
|
|
1871
1898
|
raise Pgbus::ConfigurationError, "#{name} must be a positive number, got #{numeric}" unless numeric.positive?
|
|
1872
1899
|
|
data/lib/pgbus/event.rb
CHANGED
|
@@ -23,6 +23,16 @@ module Pgbus
|
|
|
23
23
|
payload.is_a?(Hash) ? payload[key.to_s] : nil
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
+
# Nested counterpart of #[]: the first key is read as a String, later
|
|
27
|
+
# Symbol keys too (payloads are JSON-decoded); Integer indexes are kept.
|
|
28
|
+
# A non-Hash payload gives nil; an intermediate value without #dig raises
|
|
29
|
+
# TypeError, as Hash#dig does.
|
|
30
|
+
def dig(key, *rest)
|
|
31
|
+
return nil unless payload.is_a?(Hash)
|
|
32
|
+
|
|
33
|
+
payload.dig(key.to_s, *rest.map { |k| k.is_a?(Symbol) ? k.to_s : k })
|
|
34
|
+
end
|
|
35
|
+
|
|
26
36
|
def to_h
|
|
27
37
|
{
|
|
28
38
|
"event_id" => event_id,
|
|
@@ -35,8 +35,12 @@ module Pgbus
|
|
|
35
35
|
# a multi-KB log line (issue #367 minor).
|
|
36
36
|
NAME_LIMIT = 10
|
|
37
37
|
|
|
38
|
-
|
|
38
|
+
# +grace+ overrides Pgbus.configuration.health_unclaimed_grace (issue #589):
|
|
39
|
+
# seconds a visible read_ct=0 message must have been claimable before it
|
|
40
|
+
# counts as "never claimed". nil reads the config when the verdict runs.
|
|
41
|
+
def initialize(data_source, grace: nil)
|
|
39
42
|
@data_source = data_source
|
|
43
|
+
@grace = grace
|
|
40
44
|
end
|
|
41
45
|
|
|
42
46
|
# Returns a machine-readable verdict hash suitable for both interactive
|
|
@@ -81,7 +85,9 @@ module Pgbus
|
|
|
81
85
|
status: status,
|
|
82
86
|
reasons: stalled + degraded,
|
|
83
87
|
checked_at: Time.now.utc.iso8601,
|
|
84
|
-
summary: build_summary(queues, non_dlq, processes, health)
|
|
88
|
+
summary: build_summary(queues, non_dlq, processes, health).merge(
|
|
89
|
+
oldest_unread_claimable_age_sec: oldest_unread_age(drained)
|
|
90
|
+
)
|
|
85
91
|
}
|
|
86
92
|
end
|
|
87
93
|
|
|
@@ -139,7 +145,8 @@ module Pgbus
|
|
|
139
145
|
"while #{active_backlog.size} queue(s) have visible backlog: #{backlog_names(active_backlog)}"
|
|
140
146
|
elsif live_drainers.any? && drained_backlog.any? && any_unread?(drained_backlog)
|
|
141
147
|
unread = drained_backlog.select { |q| unread?(q) }
|
|
142
|
-
reasons << "#{unread.size} queue(s) have visible messages with read_ct=0
|
|
148
|
+
reasons << "#{unread.size} queue(s) have visible messages with read_ct=0 " \
|
|
149
|
+
"(never claimed#{unread_age_clause(unread)}) " \
|
|
143
150
|
"while #{live_drainers.size} draining process(es) are alive: #{backlog_names(unread)}"
|
|
144
151
|
end
|
|
145
152
|
reasons
|
|
@@ -211,10 +218,39 @@ module Pgbus
|
|
|
211
218
|
#
|
|
212
219
|
# When metrics don't expose the count (an old hash / a stubbed test) we
|
|
213
220
|
# conservatively treat the queue as unread so we never miss the wedge.
|
|
221
|
+
#
|
|
222
|
+
# Issue #589: the oldest such row must also have been claimable for at
|
|
223
|
+
# least the grace. A fresh enqueue, or a delayed job whose wait just
|
|
224
|
+
# expired, is visible with read_ct=0 until a worker's next poll, so
|
|
225
|
+
# without the grace a healthy fleet read STALLED (and /readyz 503) on a
|
|
226
|
+
# large share of checks. An unknown age (key missing or nil) stays
|
|
227
|
+
# conservative and counts.
|
|
214
228
|
def unread?(queue)
|
|
215
229
|
return true unless queue.key?(:visible_unread_length)
|
|
230
|
+
return false unless queue[:visible_unread_length].to_i.positive?
|
|
231
|
+
|
|
232
|
+
age = unread_age(queue)
|
|
233
|
+
age.nil? || age >= grace
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
def grace
|
|
237
|
+
@grace || Pgbus.configuration.health_unclaimed_grace
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def unread_age(queue)
|
|
241
|
+
queue[:oldest_unread_claimable_age_sec]
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# Oldest known unread claimable age over +queues+, nil when none is known.
|
|
245
|
+
def oldest_unread_age(queues)
|
|
246
|
+
queues.filter_map { |q| unread_age(q) }.max
|
|
247
|
+
end
|
|
216
248
|
|
|
217
|
-
|
|
249
|
+
# ", oldest claimable for 74s" so an operator can tell a wedge from a
|
|
250
|
+
# spike; empty when no flagged queue reports an age.
|
|
251
|
+
def unread_age_clause(queues)
|
|
252
|
+
age = oldest_unread_age(queues)
|
|
253
|
+
age ? ", oldest claimable for #{age}s" : ""
|
|
218
254
|
end
|
|
219
255
|
|
|
220
256
|
# Join backlog queue names for a reason string, truncated so a flagged
|
|
@@ -13,8 +13,10 @@ module Pgbus
|
|
|
13
13
|
description <<~DESC
|
|
14
14
|
Top-level pgbus health verdict. Returns OK, DEGRADED, or STALLED with
|
|
15
15
|
the specific reasons. STALLED means the silent-wedge condition:
|
|
16
|
-
queues
|
|
17
|
-
|
|
16
|
+
queues hold a visible, never-claimed backlog older than
|
|
17
|
+
health_unclaimed_grace (default 60s) while workers are heart-beating
|
|
18
|
+
but not claiming (or a worker's claim loop has stalled); the summary's
|
|
19
|
+
oldest_unread_claimable_age_sec tells a wedge from a spike. DEGRADED covers
|
|
18
20
|
stale processes, paused queues holding a backlog, dead-letter
|
|
19
21
|
buildup, or a long-running transaction pinning the MVCC horizon. Use
|
|
20
22
|
this first when triaging — it summarizes the whole system in one call
|
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
-- PGMQ 1.13.1 table fixup.
|
|
2
|
+
--
|
|
3
|
+
-- Adapted from the upstream migration
|
|
4
|
+
-- pgmq-extension/sql/pgmq--1.13.0--1.13.1.sql at tag v1.13.1
|
|
5
|
+
-- (blob 16cbbe2763ec6508570a80e7ef08b7bee3b11ffd). Only the parts pgbus's
|
|
6
|
+
-- upgrade cannot already reproduce are here: dropping every function and
|
|
7
|
+
-- composite type and re-running the target schema recreates functions and
|
|
8
|
+
-- types, but never touches an existing table, its column defaults or its
|
|
9
|
+
-- grants.
|
|
10
|
+
--
|
|
11
|
+
-- Left out on purpose: upstream's loop that re-creates each queue's NOTIFY
|
|
12
|
+
-- trigger with the queue name as its argument. The upgrade's
|
|
13
|
+
-- reinstall_notify_triggers_sql step replays the new pgmq.enable_notify_insert
|
|
14
|
+
-- for every row of pgmq.notify_insert_throttle, which re-creates the trigger
|
|
15
|
+
-- the same way.
|
|
16
|
+
--
|
|
17
|
+
-- One deliberate deviation from upstream: the rebind-defaults block selects
|
|
18
|
+
-- pgmq's tables by name in the pgmq schema instead of by extension membership
|
|
19
|
+
-- (pg_depend deptype 'e'). An embedded install has no extension, so upstream's
|
|
20
|
+
-- selection would find nothing and the block would never run. The pg_monitor
|
|
21
|
+
-- blocks are upstream's, unchanged.
|
|
22
|
+
--
|
|
23
|
+
-- Every file in this directory MUST be idempotent and self-guarded. The
|
|
24
|
+
-- upgrade migration applies each fixup between the recorded installed version
|
|
25
|
+
-- and the target, and an install whose version was never recorded gets all of
|
|
26
|
+
-- them, so a fixup can be replayed against a database that already has it.
|
|
27
|
+
|
|
28
|
+
-- Queue tables created by the old functions can retain defaults bound to a
|
|
29
|
+
-- function in a writable schema. Replacing the functions does not replace
|
|
30
|
+
-- those stored expression dependencies. Rebind only PGMQ's own defaults that
|
|
31
|
+
-- refer to a non-catalog now() or to_timestamp(). Include partition children,
|
|
32
|
+
-- since their defaults can be stored separately from the parent.
|
|
33
|
+
DO $pgmq_rebind_defaults$
|
|
34
|
+
DECLARE
|
|
35
|
+
def RECORD;
|
|
36
|
+
BEGIN
|
|
37
|
+
FOR def IN
|
|
38
|
+
WITH RECURSIVE pgmq_tables AS (
|
|
39
|
+
-- pgbus: pgmq schema tables by name, not extension members (see header).
|
|
40
|
+
SELECT c.oid, c.relname
|
|
41
|
+
FROM pg_catalog.pg_class c
|
|
42
|
+
WHERE c.relnamespace = 'pgmq'::pg_catalog.regnamespace
|
|
43
|
+
AND c.relkind IN ('r', 'p')
|
|
44
|
+
), roots AS (
|
|
45
|
+
SELECT c.oid AS relid,
|
|
46
|
+
CASE c.relname
|
|
47
|
+
WHEN 'meta' THEN 'meta'
|
|
48
|
+
WHEN 'topic_bindings' THEN 'topic_bindings'
|
|
49
|
+
ELSE 'notify_insert_throttle'
|
|
50
|
+
END AS kind
|
|
51
|
+
FROM pgmq_tables c
|
|
52
|
+
WHERE c.relname IN ('meta', 'topic_bindings', 'notify_insert_throttle')
|
|
53
|
+
UNION
|
|
54
|
+
SELECT c.oid, names.kind
|
|
55
|
+
FROM pgmq_tables core
|
|
56
|
+
JOIN pgmq.meta m ON core.relname = 'meta'
|
|
57
|
+
CROSS JOIN LATERAL (
|
|
58
|
+
VALUES (pgmq.format_table_name(m.queue_name::pg_catalog.text, 'q'::pg_catalog.text), 'queue'),
|
|
59
|
+
(pgmq.format_table_name(m.queue_name::pg_catalog.text, 'a'::pg_catalog.text), 'archive')
|
|
60
|
+
) AS names(relname, kind)
|
|
61
|
+
JOIN pg_catalog.pg_class c
|
|
62
|
+
ON c.relnamespace = 'pgmq'::pg_catalog.regnamespace
|
|
63
|
+
AND c.relname = names.relname
|
|
64
|
+
AND c.relkind IN ('r', 'p')
|
|
65
|
+
), managed AS (
|
|
66
|
+
SELECT relid, kind FROM roots
|
|
67
|
+
UNION
|
|
68
|
+
SELECT i.inhrelid, m.kind
|
|
69
|
+
FROM managed m
|
|
70
|
+
JOIN pg_catalog.pg_inherits i ON i.inhparent = m.relid
|
|
71
|
+
)
|
|
72
|
+
SELECT n.nspname, c.relname, a.attname,
|
|
73
|
+
pg_catalog.pg_get_expr(ad.adbin, ad.adrelid) AS old_expression,
|
|
74
|
+
CASE WHEN m.kind = 'notify_insert_throttle'
|
|
75
|
+
THEN 'pg_catalog.to_timestamp(0)'
|
|
76
|
+
ELSE 'pg_catalog.now()'
|
|
77
|
+
END AS replacement,
|
|
78
|
+
EXISTS (
|
|
79
|
+
SELECT 1
|
|
80
|
+
FROM pg_catalog.pg_depend d
|
|
81
|
+
JOIN pg_catalog.pg_proc p ON p.oid = d.refobjid
|
|
82
|
+
JOIN pg_catalog.pg_namespace pn ON pn.oid = p.pronamespace
|
|
83
|
+
WHERE d.classid = 'pg_catalog.pg_attrdef'::pg_catalog.regclass
|
|
84
|
+
AND d.objid = ad.oid
|
|
85
|
+
AND d.refclassid = 'pg_catalog.pg_proc'::pg_catalog.regclass
|
|
86
|
+
AND pn.nspname <> 'pg_catalog'
|
|
87
|
+
AND ((m.kind = 'notify_insert_throttle'
|
|
88
|
+
AND p.proname = 'to_timestamp'
|
|
89
|
+
AND p.proargtypes[0] = 'pg_catalog.int4'::pg_catalog.regtype
|
|
90
|
+
AND pg_catalog.pg_get_expr(ad.adbin, ad.adrelid) =
|
|
91
|
+
pg_catalog.format('%I.to_timestamp(0)', pn.nspname))
|
|
92
|
+
OR (m.kind <> 'notify_insert_throttle'
|
|
93
|
+
AND p.proname = 'now'
|
|
94
|
+
AND p.pronargs = 0
|
|
95
|
+
AND pg_catalog.pg_get_expr(ad.adbin, ad.adrelid) =
|
|
96
|
+
pg_catalog.format('%I.now()', pn.nspname)))
|
|
97
|
+
) AS known_shape
|
|
98
|
+
FROM managed m
|
|
99
|
+
JOIN pg_catalog.pg_class c ON c.oid = m.relid
|
|
100
|
+
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
|
101
|
+
JOIN pg_catalog.pg_attribute a ON a.attrelid = c.oid
|
|
102
|
+
JOIN pg_catalog.pg_attrdef ad
|
|
103
|
+
ON ad.adrelid = c.oid AND ad.adnum = a.attnum
|
|
104
|
+
WHERE ((m.kind = 'meta' AND a.attname = 'created_at')
|
|
105
|
+
OR (m.kind = 'topic_bindings' AND a.attname = 'bound_at')
|
|
106
|
+
OR (m.kind = 'notify_insert_throttle' AND a.attname = 'last_notified_at')
|
|
107
|
+
OR (m.kind = 'queue' AND a.attname = 'enqueued_at')
|
|
108
|
+
OR (m.kind = 'archive' AND a.attname IN ('enqueued_at', 'archived_at')))
|
|
109
|
+
AND EXISTS (
|
|
110
|
+
SELECT 1
|
|
111
|
+
FROM pg_catalog.pg_depend d
|
|
112
|
+
JOIN pg_catalog.pg_proc p ON p.oid = d.refobjid
|
|
113
|
+
JOIN pg_catalog.pg_namespace pn ON pn.oid = p.pronamespace
|
|
114
|
+
WHERE d.classid = 'pg_catalog.pg_attrdef'::pg_catalog.regclass
|
|
115
|
+
AND d.objid = ad.oid
|
|
116
|
+
AND d.refclassid = 'pg_catalog.pg_proc'::pg_catalog.regclass
|
|
117
|
+
AND pn.nspname <> 'pg_catalog'
|
|
118
|
+
)
|
|
119
|
+
LOOP
|
|
120
|
+
IF NOT def.known_shape THEN
|
|
121
|
+
RAISE EXCEPTION 'PGMQ upgrade cannot safely rebind custom default on %.%.%: %; inspect the non-catalog function dependency before retrying',
|
|
122
|
+
def.nspname, def.relname, def.attname, def.old_expression;
|
|
123
|
+
END IF;
|
|
124
|
+
BEGIN
|
|
125
|
+
EXECUTE pg_catalog.format(
|
|
126
|
+
'ALTER TABLE %I.%I ALTER COLUMN %I SET DEFAULT %s',
|
|
127
|
+
def.nspname, def.relname, def.attname, def.replacement
|
|
128
|
+
);
|
|
129
|
+
EXCEPTION WHEN insufficient_privilege THEN
|
|
130
|
+
RAISE EXCEPTION 'PGMQ upgrade cannot rebind default on %.%.%; use a role that can update the extension and alter the table, or a superuser',
|
|
131
|
+
def.nspname, def.relname, def.attname;
|
|
132
|
+
END;
|
|
133
|
+
END LOOP;
|
|
134
|
+
END
|
|
135
|
+
$pgmq_rebind_defaults$;
|
|
136
|
+
|
|
137
|
+
-- pg_monitor could read every table and sequence in the pgmq schema, including
|
|
138
|
+
-- queue and archive tables. Replace that broad access with the minimum needed
|
|
139
|
+
-- by pgmq.metrics(), pgmq.metrics_all(), and pgmq.list_queues().
|
|
140
|
+
|
|
141
|
+
-- 1. Remove default privileges that granted SELECT on every new table and
|
|
142
|
+
-- sequence. The grantor can differ from the role running this upgrade.
|
|
143
|
+
DO $$
|
|
144
|
+
DECLARE
|
|
145
|
+
acl RECORD;
|
|
146
|
+
BEGIN
|
|
147
|
+
FOR acl IN
|
|
148
|
+
SELECT DISTINCT d.defaclrole::pg_catalog.regrole AS role_name
|
|
149
|
+
FROM pg_catalog.pg_default_acl d
|
|
150
|
+
JOIN pg_catalog.pg_namespace n ON n.oid = d.defaclnamespace
|
|
151
|
+
CROSS JOIN LATERAL pg_catalog.aclexplode(d.defaclacl) a
|
|
152
|
+
WHERE n.nspname = 'pgmq'
|
|
153
|
+
AND d.defaclobjtype IN ('r', 'S')
|
|
154
|
+
AND a.grantee = 'pg_monitor'::pg_catalog.regrole
|
|
155
|
+
LOOP
|
|
156
|
+
BEGIN
|
|
157
|
+
EXECUTE pg_catalog.format(
|
|
158
|
+
'ALTER DEFAULT PRIVILEGES FOR ROLE %s IN SCHEMA pgmq REVOKE SELECT ON TABLES FROM pg_monitor',
|
|
159
|
+
acl.role_name
|
|
160
|
+
);
|
|
161
|
+
EXECUTE pg_catalog.format(
|
|
162
|
+
'ALTER DEFAULT PRIVILEGES FOR ROLE %s IN SCHEMA pgmq REVOKE SELECT ON SEQUENCES FROM pg_monitor',
|
|
163
|
+
acl.role_name
|
|
164
|
+
);
|
|
165
|
+
EXCEPTION WHEN insufficient_privilege OR undefined_object THEN
|
|
166
|
+
RAISE WARNING 'pgmq: could not remove default privileges of role % granting SELECT to pg_monitor in schema pgmq; run "ALTER DEFAULT PRIVILEGES FOR ROLE % IN SCHEMA pgmq REVOKE SELECT ON TABLES FROM pg_monitor" (and the same ON SEQUENCES) as that role or a superuser', acl.role_name, acl.role_name;
|
|
167
|
+
END;
|
|
168
|
+
END LOOP;
|
|
169
|
+
END;
|
|
170
|
+
$$;
|
|
171
|
+
|
|
172
|
+
-- 2. Revoke whole-relation SELECT from pg_monitor on every relation in pgmq
|
|
173
|
+
-- except meta. Step 4 reports grants that the upgrade role cannot remove.
|
|
174
|
+
DO $$
|
|
175
|
+
DECLARE
|
|
176
|
+
rel RECORD;
|
|
177
|
+
BEGIN
|
|
178
|
+
FOR rel IN
|
|
179
|
+
SELECT DISTINCT c.oid::pg_catalog.regclass AS rel_name,
|
|
180
|
+
c.relkind = 'S' AS is_sequence
|
|
181
|
+
FROM pg_catalog.pg_class c
|
|
182
|
+
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
|
183
|
+
CROSS JOIN LATERAL pg_catalog.aclexplode(c.relacl) a
|
|
184
|
+
WHERE n.nspname = 'pgmq'
|
|
185
|
+
AND c.relkind IN ('r', 'p', 'v', 'm', 'f', 'S')
|
|
186
|
+
AND c.relname <> 'meta'
|
|
187
|
+
AND a.grantee = 'pg_monitor'::pg_catalog.regrole
|
|
188
|
+
AND a.privilege_type = 'SELECT'
|
|
189
|
+
LOOP
|
|
190
|
+
BEGIN
|
|
191
|
+
EXECUTE pg_catalog.format(
|
|
192
|
+
'REVOKE SELECT ON %s %s FROM pg_monitor',
|
|
193
|
+
CASE WHEN rel.is_sequence THEN 'SEQUENCE' ELSE 'TABLE' END,
|
|
194
|
+
rel.rel_name
|
|
195
|
+
);
|
|
196
|
+
EXCEPTION WHEN insufficient_privilege THEN
|
|
197
|
+
RAISE WARNING 'pgmq: could not revoke SELECT on % from pg_monitor; run "REVOKE SELECT ON % FROM pg_monitor" as its owner or a superuser', rel.rel_name, rel.rel_name;
|
|
198
|
+
END;
|
|
199
|
+
END LOOP;
|
|
200
|
+
END;
|
|
201
|
+
$$;
|
|
202
|
+
|
|
203
|
+
-- 3. Grant the minimal metrics access on each existing queue. A grant on a
|
|
204
|
+
-- partitioned parent is sufficient because metrics query the parent.
|
|
205
|
+
DO $$
|
|
206
|
+
DECLARE
|
|
207
|
+
q RECORD;
|
|
208
|
+
qtable TEXT;
|
|
209
|
+
qseq TEXT;
|
|
210
|
+
BEGIN
|
|
211
|
+
FOR q IN SELECT queue_name FROM pgmq.meta LOOP
|
|
212
|
+
qtable := pgmq.format_table_name(
|
|
213
|
+
q.queue_name::pg_catalog.text,
|
|
214
|
+
'q'::pg_catalog.text
|
|
215
|
+
);
|
|
216
|
+
qseq := qtable || '_msg_id_seq';
|
|
217
|
+
IF pg_catalog.to_regclass(pg_catalog.format('pgmq.%I', qtable)) IS NOT NULL THEN
|
|
218
|
+
BEGIN
|
|
219
|
+
EXECUTE pg_catalog.format(
|
|
220
|
+
'GRANT SELECT (vt, enqueued_at) ON pgmq.%I TO pg_monitor',
|
|
221
|
+
qtable
|
|
222
|
+
);
|
|
223
|
+
EXCEPTION WHEN insufficient_privilege THEN
|
|
224
|
+
RAISE WARNING 'pgmq: could not grant SELECT (vt, enqueued_at) on pgmq.% to pg_monitor; run it as the table owner or a superuser so queue metrics work for pg_monitor', pg_catalog.quote_ident(qtable);
|
|
225
|
+
END;
|
|
226
|
+
END IF;
|
|
227
|
+
IF pg_catalog.to_regclass(pg_catalog.format('pgmq.%I', qseq)) IS NOT NULL THEN
|
|
228
|
+
BEGIN
|
|
229
|
+
EXECUTE pg_catalog.format(
|
|
230
|
+
'GRANT SELECT ON SEQUENCE pgmq.%I TO pg_monitor',
|
|
231
|
+
qseq
|
|
232
|
+
);
|
|
233
|
+
EXCEPTION WHEN insufficient_privilege THEN
|
|
234
|
+
RAISE WARNING 'pgmq: could not grant SELECT on sequence pgmq.% to pg_monitor; run it as the sequence owner or a superuser so queue metrics work for pg_monitor', pg_catalog.quote_ident(qseq);
|
|
235
|
+
END;
|
|
236
|
+
END IF;
|
|
237
|
+
END LOOP;
|
|
238
|
+
END;
|
|
239
|
+
$$;
|
|
240
|
+
|
|
241
|
+
-- 4. Report any heap-like relation that still exposes whole-table SELECT to
|
|
242
|
+
-- pg_monitor. Column grants live in pg_attribute and are not reported here.
|
|
243
|
+
DO $$
|
|
244
|
+
DECLARE
|
|
245
|
+
leftover TEXT;
|
|
246
|
+
BEGIN
|
|
247
|
+
SELECT pg_catalog.string_agg(
|
|
248
|
+
c.oid::pg_catalog.regclass::pg_catalog.text,
|
|
249
|
+
', ' ORDER BY c.oid::pg_catalog.regclass::pg_catalog.text
|
|
250
|
+
)
|
|
251
|
+
INTO leftover
|
|
252
|
+
FROM pg_catalog.pg_class c
|
|
253
|
+
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
|
254
|
+
CROSS JOIN LATERAL pg_catalog.aclexplode(c.relacl) a
|
|
255
|
+
WHERE n.nspname = 'pgmq'
|
|
256
|
+
AND c.relkind IN ('r', 'p', 'v', 'm', 'f')
|
|
257
|
+
AND c.relname <> 'meta'
|
|
258
|
+
AND a.grantee = 'pg_monitor'::pg_catalog.regrole
|
|
259
|
+
AND a.privilege_type = 'SELECT';
|
|
260
|
+
|
|
261
|
+
IF leftover IS NOT NULL THEN
|
|
262
|
+
RAISE WARNING 'pgmq: pg_monitor still has whole-table SELECT on: %. These may expose message payloads; revoke them as the object owner or a superuser.', leftover;
|
|
263
|
+
END IF;
|
|
264
|
+
END;
|
|
265
|
+
$$;
|