pgmq-ruby 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +105 -1
- data/README.md +299 -19
- data/lib/pgmq/client/autovacuum.rb +180 -0
- data/lib/pgmq/client/consumer.rb +121 -7
- data/lib/pgmq/client/maintenance.rb +154 -5
- data/lib/pgmq/client/message_lifecycle.rb +6 -7
- data/lib/pgmq/client/metrics.rb +1 -2
- data/lib/pgmq/client/multi_queue.rb +9 -11
- data/lib/pgmq/client/producer.rb +37 -12
- data/lib/pgmq/client/queue_management.rb +87 -5
- data/lib/pgmq/client/topics.rb +22 -12
- data/lib/pgmq/client.rb +49 -34
- data/lib/pgmq/connection.rb +139 -33
- data/lib/pgmq/message.rb +4 -5
- data/lib/pgmq/notify_throttle.rb +25 -0
- data/lib/pgmq/queue_name.rb +153 -0
- data/lib/pgmq/transaction.rb +6 -7
- data/lib/pgmq/version.rb +1 -1
- data/lib/pgmq.rb +2 -3
- metadata +5 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ee008476f444de984ef7413b21bee87043d9e5e87a7eabf85a934cf74b147225
|
|
4
|
+
data.tar.gz: 1dd10b9b77c9dc4109f6a8e779c970fc6f7ee2e4880cecb467ab3060ecc4f571
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 32cc154c5df5d6501de5bf93daf32857448645b9fe5071bec84b51dbda86cf0efd1fe6cddfd24707fda8201e7d6b8fb10c02f3c7e2bf108226b50a4cbe1b65fd
|
|
7
|
+
data.tar.gz: ad2630d6736bc49490ccb2eec83da4878dd4b349e9e06cd12f5d9f62a10109922a379c0dab43aa5a0c8a3f33241ca71f6c6ba24386cc6a7cf097db1397ef2518
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,109 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.7.0 (2026-06-15)
|
|
4
|
+
|
|
5
|
+
### Queue Naming
|
|
6
|
+
- **[Feature]** Add `PGMQ::QueueName`, a module that is now the single source of truth for queue-name rules and
|
|
7
|
+
exposes tiers for deriving valid names from less-trusted input:
|
|
8
|
+
- `PGMQ::QueueName.valid?(name)` / `PGMQ::QueueName.validate!(name)` - the existing strict check (used internally
|
|
9
|
+
by every `PGMQ::Client` operation); `validate!` returns the name when valid and raises
|
|
10
|
+
`PGMQ::Errors::InvalidQueueNameError` otherwise.
|
|
11
|
+
- `PGMQ::QueueName.normalize(name)` - rewrites a name meant to be valid but using friendly separators. Maps
|
|
12
|
+
hyphens, dots, and colons to underscores, strips any *other* invalid character (so `"a@b"` → `"ab"`, not
|
|
13
|
+
`"a_b"`), then validates - e.g. a Turbo Stream channel `"chat:room-7"` → `"chat_room_7"`. Raises if the result
|
|
14
|
+
still can't be valid (empty, or starts with a digit). Colons map to underscores (rather than being stripped) so
|
|
15
|
+
distinct turbo-rails stream names don't collide on one queue.
|
|
16
|
+
- `PGMQ::QueueName.sanitize!(name)` - strips every invalid character then validates; raises
|
|
17
|
+
`InvalidQueueNameError` if nothing valid remains. A SQL-identifier guard for untrusted input: the result is
|
|
18
|
+
always either a name you know is safe or an exception, never a silent substitute.
|
|
19
|
+
- `PGMQ::QueueName.sanitize(name)` - the lenient sibling of `sanitize!`: best-effort coercion that never raises for
|
|
20
|
+
content (lowercases, replaces illegal runs, prefixes leading digits, truncates to the length limit, falls back
|
|
21
|
+
to `"queue"`). Convenient, but distinct inputs can map to the same name, so prefer `sanitize!` for untrusted
|
|
22
|
+
input.
|
|
23
|
+
|
|
24
|
+
`PGMQ::Client#validate_queue_name!` now delegates to `PGMQ::QueueName.validate!`; error messages are unchanged.
|
|
25
|
+
### Connection Management
|
|
26
|
+
- **[Feature]** `PGMQ::Client#with_connection` is now public. Every PGMQ operation already checks
|
|
27
|
+
out a pooled, health-checked connection through this method; exposing it lets callers run
|
|
28
|
+
PostgreSQL statements PGMQ does not wrap (ad-hoc `NOTIFY`/`LISTEN`, advisory locks, custom
|
|
29
|
+
monitoring queries, DDL alongside queue tables) on the same pool instead of standing up a second
|
|
30
|
+
one. The yielded object is the raw `PG::Connection` (no type mapping, no implicit transaction);
|
|
31
|
+
use `client.transaction` when atomicity is required.
|
|
32
|
+
|
|
33
|
+
### Queue Maintenance
|
|
34
|
+
- **[Feature]** Add `list_notify_insert_throttles` — returns an array of `PGMQ::NotifyThrottle`
|
|
35
|
+
objects (one per queue with NOTIFY enabled), each carrying `queue_name`, `throttle_interval_ms`,
|
|
36
|
+
and `last_notified_at`. Useful for auditing notification configuration across all queues.
|
|
37
|
+
Requires PGMQ v1.11.0+.
|
|
38
|
+
- **[Feature]** Add `update_notify_insert(queue_name, throttle_interval_ms:)` — updates the NOTIFY
|
|
39
|
+
throttle interval on an already-enabled trigger without having to disable and re-enable it.
|
|
40
|
+
Requires PGMQ v1.11.0+.
|
|
41
|
+
- **[Feature]** Add `wait_for_notify(queue_name, timeout: nil)` — thin wrapper around PostgreSQL `LISTEN/NOTIFY`
|
|
42
|
+
for event-driven message consumption. Blocks until the queue's NOTIFY channel (`pgmq.q_<queue>.INSERT`) fires or the
|
|
43
|
+
timeout expires, then issues `UNLISTEN` and returns the connection to the pool. Unlike `read_with_poll`, which
|
|
44
|
+
holds a connection open inside a PL/pgSQL loop for the full poll window, `wait_for_notify` releases the
|
|
45
|
+
connection the moment the notification arrives — more efficient under low message rates. Requires
|
|
46
|
+
`enable_notify_insert` to be called first to attach the server-side trigger.
|
|
47
|
+
- **[Feature]** Add `convert_archive_partitioned(queue_name, partition_interval:, retention_interval:,
|
|
48
|
+
leading_partition:)` - converts a standard queue's archive table to a pg_partman-managed partitioned table.
|
|
49
|
+
Provides a migration path for queues originally created with `create` or `create_unlogged` whose archive tables
|
|
50
|
+
have grown large enough to benefit from partitioning. Idempotent: returns without error if the archive table is
|
|
51
|
+
already partitioned or does not exist. Requires the `pg_partman` PostgreSQL extension.
|
|
52
|
+
|
|
53
|
+
### Queue Management
|
|
54
|
+
- **[Feature]** Add `create_fifo_index(queue_name)` - creates the FIFO index on a queue's underlying table required
|
|
55
|
+
for correct ordering and acceptable performance with grouped read operations (`read_grouped`, `read_grouped_rr`,
|
|
56
|
+
`read_grouped_head`). The operation is idempotent.
|
|
57
|
+
- **[Feature]** Add `create_fifo_indexes_all` - convenience wrapper that creates FIFO indexes on every queue
|
|
58
|
+
registered in `pgmq.meta`. Useful for one-time migrations when adding grouped reads to an existing deployment.
|
|
59
|
+
- **[Feature]** Add `tune_autovacuum(queue_name, queue_settings:, archive:, archive_settings:)` - sets per-table
|
|
60
|
+
autovacuum and storage parameters on a queue's tables (`pgmq.q_<name>` and, unless `archive: false`,
|
|
61
|
+
`pgmq.a_<name>`) via `ALTER TABLE`. PGMQ tables churn under constant insert/update/delete (every read UPDATEs
|
|
62
|
+
`vt`/`read_ct`/`last_read_at`), so PostgreSQL's defaults (`autovacuum_vacuum_scale_factor` 0.2, `fillfactor` 100)
|
|
63
|
+
let dead tuples and index/page bloat accumulate before autovacuum runs. The tuned defaults set, on the queue
|
|
64
|
+
table: `autovacuum_vacuum_scale_factor 0.01`, `autovacuum_vacuum_threshold 50`, `autovacuum_vacuum_cost_delay 2`,
|
|
65
|
+
`autovacuum_analyze_scale_factor 0.05`, and `fillfactor 70` (the indexed per-read UPDATE is not HOT-eligible, so
|
|
66
|
+
page headroom helps); and on the archive table the same minus `fillfactor` (append-only) with `cost_delay 5`.
|
|
67
|
+
Pass `queue_settings:`/`archive_settings:` Hashes to override individual parameters (merged onto the defaults).
|
|
68
|
+
Parameter names are allow-listed and values coerced (`Float`/`Integer`); the table name is quoted (`quote_ident`)
|
|
69
|
+
and lower-cased to match PGMQ's table naming. Opt-in: the gem does not change storage parameters unless asked.
|
|
70
|
+
- **[Feature]** `create`, `create_unlogged`, and `create_partitioned` accept a `tune_autovacuum:` keyword. Pass
|
|
71
|
+
`true` to apply the tuned defaults to the new queue's tables, or a Hash of the `tune_autovacuum` keyword options.
|
|
72
|
+
Defaults to `false` (no change), preserving the thin-wrapper behaviour.
|
|
73
|
+
|
|
74
|
+
### Message Operations
|
|
75
|
+
- **[Enhancement]** Add Ruby warning category opt-in to test helpers
|
|
76
|
+
- **[Feature]** Add `read_grouped_head(queue_name, vt:, qty:)` - reads exactly one message (the
|
|
77
|
+
oldest visible) from each distinct FIFO group, up to `qty` groups. Groups are identified by the
|
|
78
|
+
`x-pgmq-group` key in message headers (set via `headers:` on `produce`); messages without that
|
|
79
|
+
header all share one implicit default group. Unlike `read_grouped` (which groups by the first
|
|
80
|
+
payload key and drains one group fully before moving on), `read_grouped_head` surfaces the
|
|
81
|
+
leading edge of every group in a single call - ideal for detecting head-of-line stalls or
|
|
82
|
+
building per-group progress dashboards. Requires PGMQ v1.11.1+.
|
|
83
|
+
- **[Feature]** Add SQS-style grouped reading:
|
|
84
|
+
- `read_grouped(queue_name, vt:, qty:)` - reads messages grouped by the first JSON key,
|
|
85
|
+
filling the batch from the oldest group first (throughput-optimised). Contrast with
|
|
86
|
+
`read_grouped_rr` which interleaves groups fairly.
|
|
87
|
+
- `read_grouped_with_poll(queue_name, vt:, qty:, max_poll_seconds:, poll_interval_ms:)` -
|
|
88
|
+
same strategy with long-polling support.
|
|
89
|
+
|
|
90
|
+
Use `read_grouped` when maximising throughput matters more than fairness across groups
|
|
91
|
+
(e.g. a single tenant has a burst of work). Use `read_grouped_rr` when you need to
|
|
92
|
+
prevent any one group from monopolising workers.
|
|
93
|
+
- **[Feature]** `produce`, `produce_batch`, and `produce_batch_topic` now accept an absolute `Time`
|
|
94
|
+
object for the `delay:` parameter in addition to an integer number of seconds. This mirrors the
|
|
95
|
+
existing `set_vt` behaviour and maps to the `timestamptz` overloads added in PGMQ v1.10.0.
|
|
96
|
+
Pass `delay: Time.now + 3600` to schedule a message to become visible at a specific wall-clock
|
|
97
|
+
time. Note: `produce_topic` does not support `Time` delay because the upstream `pgmq.send_topic`
|
|
98
|
+
SQL function does not yet have a `timestamptz` overload.
|
|
99
|
+
|
|
100
|
+
## 0.6.2 (2026-05-08)
|
|
101
|
+
|
|
102
|
+
### Connection Management
|
|
103
|
+
- **[Feature]** Add class-level configuration for extending reconnectable error detection. `PGMQ::Connection.reconnectable_error_patterns` accepts additional `String` or `Regexp` patterns (strings matched as case-insensitive substrings; regexps against the original message). `PGMQ::Connection.reconnectable_error_classes` accepts additional `Exception` subclasses. This allows users to adapt to new pg gem / PostgreSQL / pooler disconnect signatures without waiting for a gem release.
|
|
104
|
+
- **[Fix]** `PGMQ::Connection#connection_lost_error?` now detects SSL-layer teardown errors (`"PQconsumeInput() SSL error: unexpected eof while reading"`, `"SSL SYSCALL error: EOF detected"`). Observed in production behind a managed Postgres pooler: an idle connection torn down at the SSL layer caused the next enqueue to raise `PGMQ::Errors::ConnectionError` without triggering `with_connection`'s single-retry path, because the message didn't match any of the existing substrings.
|
|
105
|
+
- **[Fix]** `PGMQ::Connection#connection_lost_error?` now also matches by class (`PG::ConnectionBad`, `PG::UnableToSend`) in addition to message substrings. These are dedicated connection-failure classes libpq raises when the socket is dead; class-matching catches future OS/pooler/TLS message variants without waiting for them to hit production.
|
|
106
|
+
|
|
3
107
|
## 0.6.1 (2026-04-16)
|
|
4
108
|
|
|
5
109
|
### Connection Management
|
|
@@ -17,7 +121,7 @@
|
|
|
17
121
|
### Infrastructure
|
|
18
122
|
- **[Change]** Migrate test framework from RSpec to Minitest/Spec with Mocha for mocking, aligning with the broader Karafka ecosystem conventions.
|
|
19
123
|
- **[Change]** Replace `rubocop-rspec` with `rubocop-minitest` for test linting.
|
|
20
|
-
- **[Change]** Add `bin/integrations` runner script that centralizes integration spec execution. Specs no longer need `require_relative "support/example_helper"`
|
|
124
|
+
- **[Change]** Add `bin/integrations` runner script that centralizes integration spec execution. Specs no longer need `require_relative "support/example_helper"` - the runner injects it via `-r` flag. Run all specs with `bin/integrations` or specific ones with `bin/integrations spec/integration/foo_spec.rb`.
|
|
21
125
|
|
|
22
126
|
## 0.5.0 (2026-02-24)
|
|
23
127
|
|
data/README.md
CHANGED
|
@@ -28,7 +28,7 @@ PGMQ-Ruby is a Ruby client for PGMQ (PostgreSQL Message Queue). It provides dire
|
|
|
28
28
|
- [Queue Management](#queue-management)
|
|
29
29
|
- [Sending Messages](#sending-messages)
|
|
30
30
|
- [Reading Messages](#reading-messages)
|
|
31
|
-
- [Grouped
|
|
31
|
+
- [Grouped Reading](#grouped-reading)
|
|
32
32
|
- [Message Lifecycle](#message-lifecycle)
|
|
33
33
|
- [Monitoring](#monitoring)
|
|
34
34
|
- [Transaction Support](#transaction-support)
|
|
@@ -49,8 +49,11 @@ This gem provides complete support for all core PGMQ SQL functions. Based on the
|
|
|
49
49
|
| **Reading** | `read` | Read single message with visibility timeout | ✅ |
|
|
50
50
|
| | `read_batch` | Read multiple messages with visibility timeout | ✅ |
|
|
51
51
|
| | `read_with_poll` | Long-polling for efficient message consumption | ✅ |
|
|
52
|
+
| | `read_grouped` | SQS-style throughput-first grouped reading | ✅ |
|
|
53
|
+
| | `read_grouped_with_poll` | Throughput-first grouped reading with long-polling | ✅ |
|
|
52
54
|
| | `read_grouped_rr` | Round-robin reading across message groups | ✅ |
|
|
53
55
|
| | `read_grouped_rr_with_poll` | Round-robin with long-polling | ✅ |
|
|
56
|
+
| | `read_grouped_head` | One message per FIFO group from the head of each group | ✅ |
|
|
54
57
|
| | `pop` | Atomic read + delete operation | ✅ |
|
|
55
58
|
| | `pop_batch` | Atomic batch read + delete operation | ✅ |
|
|
56
59
|
| **Deleting/Archiving** | `delete` | Delete single message | ✅ |
|
|
@@ -58,9 +61,12 @@ This gem provides complete support for all core PGMQ SQL functions. Based on the
|
|
|
58
61
|
| | `archive` | Archive single message for long-term storage | ✅ |
|
|
59
62
|
| | `archive_batch` | Archive multiple messages | ✅ |
|
|
60
63
|
| | `purge_queue` | Remove all messages from queue | ✅ |
|
|
64
|
+
| | `convert_archive_partitioned` | Convert archive table to pg_partman-managed partitions | ✅ |
|
|
61
65
|
| **Queue Management** | `create` | Create standard queue | ✅ |
|
|
62
66
|
| | `create_partitioned` | Create partitioned queue (requires pg_partman) | ✅ |
|
|
63
67
|
| | `create_unlogged` | Create unlogged queue (faster, no crash recovery) | ✅ |
|
|
68
|
+
| | `create_fifo_index` | Create FIFO index required for grouped reads | ✅ |
|
|
69
|
+
| | `create_fifo_indexes_all` | Create FIFO indexes on all existing queues | ✅ |
|
|
64
70
|
| | `drop_queue` | Delete queue and all messages | ✅ |
|
|
65
71
|
| **Topic Routing** | `bind_topic` | Bind topic pattern to queue (AMQP-like) | ✅ |
|
|
66
72
|
| | `unbind_topic` | Remove topic binding | ✅ |
|
|
@@ -75,7 +81,10 @@ This gem provides complete support for all core PGMQ SQL functions. Based on the
|
|
|
75
81
|
| | `metrics` | Get queue metrics (length, age, total messages) | ✅ |
|
|
76
82
|
| | `metrics_all` | Get metrics for all queues | ✅ |
|
|
77
83
|
| | `enable_notify_insert` | Enable PostgreSQL NOTIFY on insert | ✅ |
|
|
84
|
+
| | `update_notify_insert` | Update throttle interval on an existing NOTIFY trigger | ✅ |
|
|
78
85
|
| | `disable_notify_insert` | Disable notifications | ✅ |
|
|
86
|
+
| | `list_notify_insert_throttles` | List all queues with NOTIFY enabled and their throttle config | ✅ |
|
|
87
|
+
| | `wait_for_notify` | Block until a NOTIFY arrives on the queue's channel | ✅ |
|
|
79
88
|
| **Ruby Enhancements** | Transaction Support | Atomic operations via `client.transaction do \|txn\|` | ✅ |
|
|
80
89
|
| | Conditional Filtering | Server-side JSONB filtering with `conditional:` | ✅ |
|
|
81
90
|
| | Multi-Queue Ops | Read/pop/delete/archive from multiple queues | ✅ |
|
|
@@ -290,10 +299,80 @@ client = PGMQ::Client.new(
|
|
|
290
299
|
)
|
|
291
300
|
```
|
|
292
301
|
|
|
302
|
+
#### Running custom SQL on the PGMQ pool
|
|
303
|
+
|
|
304
|
+
Every PGMQ operation checks out a pooled connection through `client.with_connection`.
|
|
305
|
+
The method is public, so you can run PostgreSQL statements PGMQ does not wrap
|
|
306
|
+
without standing up a second connection pool. The connection is health-checked
|
|
307
|
+
(when `auto_reconnect` is enabled) and returned to the pool when the block exits.
|
|
308
|
+
|
|
309
|
+
```ruby
|
|
310
|
+
# Fire a custom NOTIFY alongside your queues
|
|
311
|
+
client.with_connection do |conn|
|
|
312
|
+
conn.exec_params("SELECT pg_notify($1, $2)", ["my_channel", payload])
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
# Run a monitoring query PGMQ does not wrap
|
|
316
|
+
depth = client.with_connection do |conn|
|
|
317
|
+
conn.exec("SELECT count(*) FROM pgmq.q_orders")[0]["count"].to_i
|
|
318
|
+
end
|
|
319
|
+
```
|
|
320
|
+
|
|
321
|
+
You receive the raw `PG::Connection`: results come back as strings (no type
|
|
322
|
+
mapping) and the statement is **not** wrapped in a transaction. Use
|
|
323
|
+
[`client.transaction`](#transaction-support) when you need atomicity.
|
|
324
|
+
|
|
325
|
+
> **Pool-safety caveats.** You're handed a *pooled* connection, so two rules apply:
|
|
326
|
+
>
|
|
327
|
+
> 1. **Don't keep the connection past the block.** Once the block returns, the
|
|
328
|
+
> connection goes back to the pool and another thread may check it out.
|
|
329
|
+
> `PG::Connection` is not thread-safe — using it afterwards can corrupt libpq
|
|
330
|
+
> state (nil results, wrong data, segfaults).
|
|
331
|
+
> 2. **Clean up session state before the block exits.** The pool does *not* reset
|
|
332
|
+
> connections on check-in. A `LISTEN`, `SET`, session-level advisory lock
|
|
333
|
+
> (`pg_advisory_lock`), prepared statement, or temp table you create survives
|
|
334
|
+
> and leaks to the next pool user. Undo it (`UNLISTEN`, `RESET`,
|
|
335
|
+
> `pg_advisory_unlock`, …) before returning. For LISTEN/NOTIFY consumption,
|
|
336
|
+
> prefer `client.wait_for_notify`, which manages `LISTEN`/`UNLISTEN` for you.
|
|
337
|
+
|
|
338
|
+
#### Extending the lost-connection error matchers
|
|
339
|
+
|
|
340
|
+
PGMQ-Ruby ships with a curated list of `PG::Error` messages and classes
|
|
341
|
+
(`PG::ConnectionBad`, `PG::UnableToSend`) that trigger the auto-reconnect
|
|
342
|
+
retry. Different `pg` gem versions, PostgreSQL versions, and connection
|
|
343
|
+
poolers (PgBouncer, Supabase, RDS Proxy, etc.) occasionally surface new
|
|
344
|
+
disconnect signatures. Rather than wait for an upstream patch, you can
|
|
345
|
+
extend the matchers at boot time via class-level configuration:
|
|
346
|
+
|
|
347
|
+
```ruby
|
|
348
|
+
# In an initializer (e.g. config/initializers/pgmq.rb for Rails apps)
|
|
349
|
+
# Strings are matched as case-insensitive substrings against the error
|
|
350
|
+
# message; Regexps are matched against the original message.
|
|
351
|
+
PGMQ::Connection.reconnectable_error_patterns = [
|
|
352
|
+
"connection reset by peer",
|
|
353
|
+
/\Abroken pipe\b/i
|
|
354
|
+
]
|
|
355
|
+
|
|
356
|
+
# Any Exception subclass is accepted. Subclasses also match.
|
|
357
|
+
PGMQ::Connection.reconnectable_error_classes = [PG::ConnectionRefused]
|
|
358
|
+
```
|
|
359
|
+
|
|
360
|
+
The built-in defaults are always kept - your patterns and classes are
|
|
361
|
+
appended to them. Configuration errors (e.g. passing an Integer as a
|
|
362
|
+
pattern) raise `PGMQ::Errors::ConfigurationError` immediately so
|
|
363
|
+
misconfiguration can't silently disable retries.
|
|
364
|
+
|
|
365
|
+
> **Reserve these options for connection-level failures.** A "reconnectable"
|
|
366
|
+
> error means the socket is dead and a retry on a fresh connection is safe.
|
|
367
|
+
> Do **not** add patterns or classes for query-level errors (deadlocks,
|
|
368
|
+
> constraint violations, statement timeouts) - those will replay your
|
|
369
|
+
> operation against a healthy connection and may cause duplicate work or
|
|
370
|
+
> mask bugs.
|
|
371
|
+
|
|
293
372
|
**Connection Pool Benefits:**
|
|
294
373
|
- **Thread-safe** - Multiple threads can safely share a single client
|
|
295
374
|
- **Fiber-aware** - Works with Ruby 3.0+ Fiber Scheduler for non-blocking I/O (tested with the `async` gem)
|
|
296
|
-
- **Auto-reconnect** - Recovers from lost connections (configurable)
|
|
375
|
+
- **Auto-reconnect** - Recovers from lost connections (configurable, extendable)
|
|
297
376
|
- **Health checks** - Verifies connections before use to prevent stale connection errors
|
|
298
377
|
- **Monitoring** - Track pool utilization with `client.stats`
|
|
299
378
|
|
|
@@ -316,9 +395,19 @@ client.create_partitioned("queue_name",
|
|
|
316
395
|
# Create unlogged queue (faster, no crash recovery)
|
|
317
396
|
client.create_unlogged("queue_name") # => true/false
|
|
318
397
|
|
|
398
|
+
# Create a queue with autovacuum tuned for PGMQ's read+delete churn (see "Autovacuum Tuning")
|
|
399
|
+
client.create("queue_name", tune_autovacuum: true)
|
|
400
|
+
|
|
319
401
|
# Drop queue (returns true if dropped, false if didn't exist)
|
|
320
402
|
client.drop_queue("queue_name") # => true/false
|
|
321
403
|
|
|
404
|
+
# Create the FIFO index required for grouped reads (read_grouped, read_grouped_rr, read_grouped_head)
|
|
405
|
+
# Idempotent - safe to call on a queue that already has the index
|
|
406
|
+
client.create_fifo_index("queue_name")
|
|
407
|
+
|
|
408
|
+
# Create FIFO indexes for all existing queues at once (useful for migrating an existing deployment)
|
|
409
|
+
client.create_fifo_indexes_all
|
|
410
|
+
|
|
322
411
|
# List all queues
|
|
323
412
|
queues = client.list_queues
|
|
324
413
|
# => [#<PGMQ::QueueMetadata queue_name="orders" created_at=...>, ...]
|
|
@@ -353,6 +442,98 @@ client.create("a" * 48) # ✗ Too long (48+ chars)
|
|
|
353
442
|
# Raises PGMQ::Errors::InvalidQueueNameError
|
|
354
443
|
```
|
|
355
444
|
|
|
445
|
+
**Deriving valid names with `PGMQ::QueueName`**
|
|
446
|
+
|
|
447
|
+
`PGMQ::Client` always validates the name you pass (via `PGMQ::QueueName.validate!`).
|
|
448
|
+
When the name comes from a friendlier source — a Turbo Stream channel, a slug, or
|
|
449
|
+
untrusted user input — `PGMQ::QueueName` gives you a few tiers so you can decide
|
|
450
|
+
how strict to be:
|
|
451
|
+
|
|
452
|
+
```ruby
|
|
453
|
+
# Tier 1 — assert a name you control is valid (raises otherwise)
|
|
454
|
+
PGMQ::QueueName.valid?("orders") # => true
|
|
455
|
+
PGMQ::QueueName.validate!("my-queue") # => raises PGMQ::Errors::InvalidQueueNameError
|
|
456
|
+
|
|
457
|
+
# Tier 2 — normalize a name meant to be valid but using friendly separators.
|
|
458
|
+
# Hyphens/dots/colons become underscores; any OTHER invalid char is stripped;
|
|
459
|
+
# raises if the result still can't be valid (empty, or starts with a digit).
|
|
460
|
+
PGMQ::QueueName.normalize("chat:room-7") # => "chat_room_7"
|
|
461
|
+
PGMQ::QueueName.normalize("order.events") # => "order_events"
|
|
462
|
+
PGMQ::QueueName.normalize("a@b") # => "ab" (the "@" is dropped, not turned into "_")
|
|
463
|
+
PGMQ::QueueName.normalize("123-go") # => raises (starts with a digit)
|
|
464
|
+
|
|
465
|
+
# Tier 3 — sanitize! untrusted input: strip every invalid char, then validate.
|
|
466
|
+
# Raises rather than substituting, so it never silently points at a different
|
|
467
|
+
# queue. Use this as a SQL-identifier guard for untrusted input.
|
|
468
|
+
PGMQ::QueueName.sanitize!("orders!!") # => "orders"
|
|
469
|
+
PGMQ::QueueName.sanitize!("!!!") # => raises PGMQ::Errors::InvalidQueueNameError
|
|
470
|
+
|
|
471
|
+
# Tier 3 (lenient) — sanitize never raises; always returns a valid name (prefixes
|
|
472
|
+
# leading digits, truncates to length, falls back to "queue"). Convenient, but
|
|
473
|
+
# distinct inputs can map to the SAME name, so prefer sanitize! for untrusted input.
|
|
474
|
+
PGMQ::QueueName.sanitize("99 Problems!") # => "q_99_problems"
|
|
475
|
+
PGMQ::QueueName.sanitize("!!!") # => "queue"
|
|
476
|
+
|
|
477
|
+
client.create(PGMQ::QueueName.sanitize!(params[:topic])) # raises on junk rather than guessing
|
|
478
|
+
```
|
|
479
|
+
|
|
480
|
+
#### Autovacuum Tuning
|
|
481
|
+
|
|
482
|
+
PGMQ tables churn in a way PostgreSQL's defaults are not tuned for. A hot queue
|
|
483
|
+
inserts, updates, and deletes rows constantly: every read UPDATEs `vt`,
|
|
484
|
+
`read_ct`, and `last_read_at`, and every read+archive cycle deletes from the
|
|
485
|
+
queue table and inserts into the archive. Two defaults hurt:
|
|
486
|
+
|
|
487
|
+
- `autovacuum_vacuum_scale_factor` defaults to `0.2`, so autovacuum only runs
|
|
488
|
+
once dead tuples reach 20% of the table — by which point a busy queue has
|
|
489
|
+
bloated its heap and B-tree indexes, slowing every read and lock.
|
|
490
|
+
- `fillfactor` defaults to `100`, so heap pages fill completely. Because `vt` is
|
|
491
|
+
indexed and changes on every read, those UPDATEs are not HOT-eligible; leaving
|
|
492
|
+
page headroom reduces page density between vacuum passes.
|
|
493
|
+
|
|
494
|
+
`tune_autovacuum` sets per-table storage parameters via `ALTER TABLE`, so
|
|
495
|
+
autovacuum runs far more often (and fillfactor reserves churn headroom) on
|
|
496
|
+
*these specific tables* without touching cluster-wide settings. It is **opt-in**
|
|
497
|
+
— the gem never mutates storage parameters unless you ask.
|
|
498
|
+
|
|
499
|
+
The tuned defaults:
|
|
500
|
+
|
|
501
|
+
| Parameter | Queue table (`pgmq.q_…`) | Archive table (`pgmq.a_…`) |
|
|
502
|
+
|-----------|--------------------------|----------------------------|
|
|
503
|
+
| `autovacuum_vacuum_scale_factor` | `0.01` | `0.05` |
|
|
504
|
+
| `autovacuum_vacuum_threshold` | `50` | `50` |
|
|
505
|
+
| `autovacuum_vacuum_cost_delay` | `2` | `5` |
|
|
506
|
+
| `autovacuum_analyze_scale_factor` | `0.05` | `0.05` |
|
|
507
|
+
| `fillfactor` | `70` | — (append-only, not set) |
|
|
508
|
+
|
|
509
|
+
```ruby
|
|
510
|
+
# Tune an existing queue with the PGMQ-tuned defaults above
|
|
511
|
+
client.tune_autovacuum("orders")
|
|
512
|
+
|
|
513
|
+
# Override individual parameters (merged onto the defaults) and skip the archive
|
|
514
|
+
client.tune_autovacuum("orders",
|
|
515
|
+
queue_settings: { autovacuum_vacuum_scale_factor: 0.005, fillfactor: 80 },
|
|
516
|
+
archive: false)
|
|
517
|
+
|
|
518
|
+
# Override an archive parameter only
|
|
519
|
+
client.tune_autovacuum("orders", archive_settings: { autovacuum_vacuum_scale_factor: 0.02 })
|
|
520
|
+
|
|
521
|
+
# Or tune at creation time (true = defaults, or a Hash of the keywords above)
|
|
522
|
+
client.create("orders", tune_autovacuum: true)
|
|
523
|
+
client.create("orders", tune_autovacuum: { queue_settings: { fillfactor: 80 }, archive: false })
|
|
524
|
+
client.create_unlogged("fast", tune_autovacuum: true)
|
|
525
|
+
client.create_partitioned("big", partition_interval: "daily",
|
|
526
|
+
retention_interval: "7 days", tune_autovacuum: true)
|
|
527
|
+
```
|
|
528
|
+
|
|
529
|
+
Only parameters you name in `queue_settings:` / `archive_settings:` change; the
|
|
530
|
+
rest keep the tuned defaults. Values are coerced (`Float`/`Integer`) and
|
|
531
|
+
parameter names are allow-listed before they reach the `ALTER TABLE`.
|
|
532
|
+
|
|
533
|
+
> **Partitioned queues:** parameters are set on the partitioned *parent* table.
|
|
534
|
+
> PostgreSQL does not cascade storage parameters to existing partitions, so set
|
|
535
|
+
> them per-partition if you need to retune already-created partitions.
|
|
536
|
+
|
|
356
537
|
### Sending Messages
|
|
357
538
|
|
|
358
539
|
```ruby
|
|
@@ -403,12 +584,38 @@ msg = client.read_with_poll("queue_name",
|
|
|
403
584
|
poll_interval_ms: 100
|
|
404
585
|
)
|
|
405
586
|
|
|
587
|
+
# Event-driven consumption via LISTEN/NOTIFY (more efficient than long-polling at low message rates)
|
|
588
|
+
# Step 1: enable server-side NOTIFY trigger once (idempotent)
|
|
589
|
+
client.enable_notify_insert("queue_name")
|
|
590
|
+
|
|
591
|
+
# Step 2: consumer loop — wake up on notification, then drain all available messages.
|
|
592
|
+
# With throttling (default 250ms), a burst of inserts fires only one NOTIFY.
|
|
593
|
+
# Always use read_batch after waking up to avoid leaving messages stranded.
|
|
594
|
+
loop do
|
|
595
|
+
next unless client.wait_for_notify("queue_name", timeout: 5)
|
|
596
|
+
|
|
597
|
+
msgs = client.read_batch("queue_name", vt: 30, qty: 10)
|
|
598
|
+
msgs.each { |m| process(m); client.delete("queue_name", m.msg_id) }
|
|
599
|
+
end
|
|
600
|
+
|
|
406
601
|
# Pop (atomic read + delete)
|
|
407
602
|
msg = client.pop("queue_name")
|
|
408
603
|
|
|
409
604
|
# Pop batch (atomic read + delete for multiple messages)
|
|
410
605
|
messages = client.pop_batch("queue_name", 10)
|
|
411
606
|
|
|
607
|
+
# SQS-style grouped reading (throughput-first: drains oldest group before moving on)
|
|
608
|
+
# Messages grouped by first key in their JSON payload
|
|
609
|
+
messages = client.read_grouped("queue_name", vt: 30, qty: 10)
|
|
610
|
+
|
|
611
|
+
# Throughput-first grouped reading with long-polling
|
|
612
|
+
messages = client.read_grouped_with_poll("queue_name",
|
|
613
|
+
vt: 30,
|
|
614
|
+
qty: 10,
|
|
615
|
+
max_poll_seconds: 5,
|
|
616
|
+
poll_interval_ms: 100
|
|
617
|
+
)
|
|
618
|
+
|
|
412
619
|
# Grouped round-robin reading (fair processing across entities)
|
|
413
620
|
# Messages are grouped by the first key in their JSON payload
|
|
414
621
|
messages = client.read_grouped_rr("queue_name", vt: 30, qty: 10)
|
|
@@ -420,38 +627,82 @@ messages = client.read_grouped_rr_with_poll("queue_name",
|
|
|
420
627
|
max_poll_seconds: 5,
|
|
421
628
|
poll_interval_ms: 100
|
|
422
629
|
)
|
|
630
|
+
|
|
631
|
+
# Read one message per FIFO group from the head of each group
|
|
632
|
+
# Groups are set via x-pgmq-group header (requires PGMQ v1.11.1+)
|
|
633
|
+
client.produce("queue_name", '{"job":"a"}', headers: '{"x-pgmq-group":"tenant_a"}')
|
|
634
|
+
client.produce("queue_name", '{"job":"b"}', headers: '{"x-pgmq-group":"tenant_b"}')
|
|
635
|
+
messages = client.read_grouped_head("queue_name", vt: 30, qty: 10)
|
|
636
|
+
# => one message from tenant_a and one from tenant_b
|
|
423
637
|
```
|
|
424
638
|
|
|
425
|
-
#### Grouped
|
|
639
|
+
#### Grouped Reading
|
|
640
|
+
|
|
641
|
+
PGMQ provides three grouped-reading strategies for processing messages from multiple entities (users, tenants, orders). All three group by the **first key** in the JSON payload (except `read_grouped_head`, which uses the `x-pgmq-group` message header).
|
|
426
642
|
|
|
427
|
-
|
|
643
|
+
##### Throughput-First (`read_grouped` / `read_grouped_with_poll`)
|
|
428
644
|
|
|
429
|
-
|
|
645
|
+
Drains the oldest group completely before moving to the next. Best when maximising throughput matters more than fairness:
|
|
430
646
|
|
|
431
647
|
```ruby
|
|
432
|
-
# Queue
|
|
433
|
-
|
|
648
|
+
# Queue: user_a has 3 messages, user_b has 1
|
|
649
|
+
messages = client.read_grouped("tasks", vt: 30, qty: 3)
|
|
650
|
+
# => [user_a_1, user_a_2, user_a_3] - drains user_a first
|
|
651
|
+
|
|
652
|
+
# With long-polling (waits up to max_poll_seconds if queue is empty)
|
|
653
|
+
messages = client.read_grouped_with_poll("tasks",
|
|
654
|
+
vt: 30,
|
|
655
|
+
qty: 10,
|
|
656
|
+
max_poll_seconds: 5,
|
|
657
|
+
poll_interval_ms: 100
|
|
658
|
+
)
|
|
659
|
+
```
|
|
434
660
|
|
|
435
|
-
|
|
436
|
-
messages = client.read_batch("tasks", vt: 30, qty: 8)
|
|
437
|
-
# => [user_a_1, user_a_2, user_a_3, user_a_4, user_a_5, user_b_1, user_b_2, user_c_1]
|
|
661
|
+
##### Round-Robin (`read_grouped_rr` / `read_grouped_rr_with_poll`)
|
|
438
662
|
|
|
439
|
-
|
|
663
|
+
Interleaves one message per group on each pass. Best for fairness - prevents any single entity from monopolising workers:
|
|
664
|
+
|
|
665
|
+
```ruby
|
|
666
|
+
# Queue: user_a: 5 messages, user_b: 2, user_c: 1
|
|
440
667
|
messages = client.read_grouped_rr("tasks", vt: 30, qty: 8)
|
|
441
668
|
# => [user_a_1, user_b_1, user_c_1, user_a_2, user_b_2, user_a_3, user_a_4, user_a_5]
|
|
442
|
-
```
|
|
443
669
|
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
670
|
+
# With long-polling
|
|
671
|
+
messages = client.read_grouped_rr_with_poll("tasks",
|
|
672
|
+
vt: 30,
|
|
673
|
+
qty: 10,
|
|
674
|
+
max_poll_seconds: 5,
|
|
675
|
+
poll_interval_ms: 100
|
|
676
|
+
)
|
|
677
|
+
```
|
|
448
678
|
|
|
449
|
-
**Message format for grouping
|
|
679
|
+
**Message format for payload-based grouping** (used by `read_grouped` and `read_grouped_rr`):
|
|
450
680
|
```ruby
|
|
451
|
-
#
|
|
681
|
+
# user_id is first key - PGMQ uses it as the group identifier
|
|
452
682
|
client.produce("tasks", '{"user_id":"user_a","task":"process"}')
|
|
683
|
+
```
|
|
453
684
|
|
|
454
|
-
|
|
685
|
+
##### Head-of-Group (`read_grouped_head`) - PGMQ v1.11.1+
|
|
686
|
+
|
|
687
|
+
Returns the oldest visible message from each FIFO group, up to `qty` groups. Groups are identified by the `x-pgmq-group` key in the message **headers**. Messages without that header all share one implicit default group.
|
|
688
|
+
|
|
689
|
+
Useful for detecting head-of-line stalls or building per-group progress dashboards - one call surfaces the leading edge of every group simultaneously:
|
|
690
|
+
|
|
691
|
+
```ruby
|
|
692
|
+
# Produce with x-pgmq-group headers
|
|
693
|
+
client.produce("jobs", '{"task":"build"}', headers: '{"x-pgmq-group":"tenant_a"}')
|
|
694
|
+
client.produce("jobs", '{"task":"build"}', headers: '{"x-pgmq-group":"tenant_a"}')
|
|
695
|
+
client.produce("jobs", '{"task":"test"}', headers: '{"x-pgmq-group":"tenant_b"}')
|
|
696
|
+
|
|
697
|
+
# Returns one message per group (oldest from each)
|
|
698
|
+
messages = client.read_grouped_head("jobs", vt: 30, qty: 100)
|
|
699
|
+
# => [tenant_a oldest msg, tenant_b oldest msg]
|
|
700
|
+
|
|
701
|
+
# Check for stuck groups
|
|
702
|
+
messages.each do |msg|
|
|
703
|
+
group = JSON.parse(msg.headers)["x-pgmq-group"]
|
|
704
|
+
puts "#{group} head enqueued at #{msg.enqueued_at}"
|
|
705
|
+
end
|
|
455
706
|
```
|
|
456
707
|
|
|
457
708
|
#### Conditional Message Filtering
|
|
@@ -538,11 +789,40 @@ client.set_vt_multi({
|
|
|
538
789
|
# Purge all messages
|
|
539
790
|
count = client.purge_queue("queue_name")
|
|
540
791
|
|
|
792
|
+
# Convert a standard queue's archive table to pg_partman-managed partitions (requires pg_partman)
|
|
793
|
+
# Useful for queues created with `create`/`create_unlogged` whose archives have grown large.
|
|
794
|
+
# Idempotent - safe to call if the archive is already partitioned or doesn't exist yet.
|
|
795
|
+
client.convert_archive_partitioned("queue_name")
|
|
796
|
+
|
|
797
|
+
# Custom partition/retention intervals (same syntax as create_partitioned)
|
|
798
|
+
client.convert_archive_partitioned("queue_name",
|
|
799
|
+
partition_interval: "daily",
|
|
800
|
+
retention_interval: "30 days"
|
|
801
|
+
)
|
|
802
|
+
|
|
541
803
|
# Enable PostgreSQL NOTIFY for a queue (for LISTEN-based consumers)
|
|
542
804
|
client.enable_notify_insert("queue_name", throttle_interval_ms: 250)
|
|
543
805
|
|
|
806
|
+
# Update the throttle interval without disabling/re-enabling the trigger
|
|
807
|
+
client.update_notify_insert("queue_name", throttle_interval_ms: 100)
|
|
808
|
+
|
|
809
|
+
# List all queues with NOTIFY enabled and their current throttle configuration
|
|
810
|
+
throttles = client.list_notify_insert_throttles
|
|
811
|
+
throttles.each do |t|
|
|
812
|
+
puts "#{t.queue_name}: #{t.throttle_interval_ms}ms (last notified: #{t.last_notified_at})"
|
|
813
|
+
end
|
|
814
|
+
|
|
544
815
|
# Disable notifications
|
|
545
816
|
client.disable_notify_insert("queue_name")
|
|
817
|
+
|
|
818
|
+
# Block until a NOTIFY arrives on the queue's channel (or timeout expires)
|
|
819
|
+
# Returns the payload string on notification, nil on timeout
|
|
820
|
+
client.wait_for_notify("queue_name", timeout: 5)
|
|
821
|
+
|
|
822
|
+
# Block form — inspect notification metadata
|
|
823
|
+
client.wait_for_notify("queue_name", timeout: 5) do |channel, pid, payload|
|
|
824
|
+
puts "Notified on #{channel} by backend #{pid}"
|
|
825
|
+
end
|
|
546
826
|
```
|
|
547
827
|
|
|
548
828
|
### Monitoring
|