pgbus 0.9.7 → 0.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +367 -0
- data/README.md +454 -25
- data/Rakefile +10 -1
- data/app/helpers/pgbus/application_helper.rb +37 -0
- data/app/views/pgbus/processes/_processes_table.html.erb +4 -1
- data/config/locales/da.yml +4 -0
- data/config/locales/de.yml +4 -0
- data/config/locales/en.yml +4 -0
- data/config/locales/es.yml +4 -0
- data/config/locales/fi.yml +4 -0
- data/config/locales/fr.yml +4 -0
- data/config/locales/it.yml +4 -0
- data/config/locales/ja.yml +4 -0
- data/config/locales/nb.yml +4 -0
- data/config/locales/nl.yml +4 -0
- data/config/locales/pt.yml +4 -0
- data/config/locales/sv.yml +4 -0
- data/lib/generators/pgbus/{add_job_locks_generator.rb → add_uniqueness_keys_generator.rb} +5 -5
- data/lib/pgbus/active_job/adapter.rb +1 -1
- data/lib/pgbus/active_job/executor.rb +25 -4
- data/lib/pgbus/cli/dlq.rb +164 -0
- data/lib/pgbus/cli.rb +18 -1
- data/lib/pgbus/client/connection_health.rb +194 -0
- data/lib/pgbus/client.rb +592 -73
- data/lib/pgbus/config_loader.rb +50 -4
- data/lib/pgbus/configuration.rb +294 -79
- data/lib/pgbus/dedup_cache.rb +8 -0
- data/lib/pgbus/doctor.rb +275 -0
- data/lib/pgbus/engine.rb +15 -0
- data/lib/pgbus/execution_pools/async_pool.rb +9 -2
- data/lib/pgbus/execution_pools/thread_pool.rb +7 -0
- data/lib/pgbus/generators/config_converter.rb +7 -5
- data/lib/pgbus/generators/migration_detector.rb +20 -16
- data/lib/pgbus/instrumentation.rb +3 -1
- data/lib/pgbus/integrations/appsignal/probe.rb +23 -1
- data/lib/pgbus/integrations/appsignal/subscriber.rb +17 -2
- data/lib/pgbus/metrics/backend.rb +38 -0
- data/lib/pgbus/metrics/backends/prometheus.rb +123 -0
- data/lib/pgbus/metrics/backends/statsd.rb +64 -0
- data/lib/pgbus/metrics/prometheus_exporter.rb +34 -0
- data/lib/pgbus/metrics/subscriber.rb +214 -0
- data/lib/pgbus/metrics.rb +43 -0
- data/lib/pgbus/pgmq_schema/pgmq_v1.11.1.sql +2126 -0
- data/lib/pgbus/pgmq_schema.rb +7 -2
- data/lib/pgbus/process/consumer.rb +354 -18
- data/lib/pgbus/process/consumer_priority.rb +34 -0
- data/lib/pgbus/process/dispatcher.rb +265 -41
- data/lib/pgbus/process/heartbeat.rb +18 -5
- data/lib/pgbus/process/memory_usage.rb +48 -0
- data/lib/pgbus/process/notify_listener.rb +26 -7
- data/lib/pgbus/process/notify_probe.rb +96 -0
- data/lib/pgbus/process/primary_validator.rb +53 -0
- data/lib/pgbus/process/signal_handler.rb +6 -0
- data/lib/pgbus/process/supervisor.rb +423 -50
- data/lib/pgbus/process/worker.rb +288 -35
- data/lib/pgbus/recurring/schedule.rb +1 -2
- data/lib/pgbus/recurring/scheduler.rb +15 -1
- data/lib/pgbus/serializer.rb +4 -4
- data/lib/pgbus/streams/broadcastable_override.rb +0 -8
- data/lib/pgbus/streams/signed_name.rb +2 -2
- data/lib/pgbus/streams/turbo_broadcastable.rb +7 -5
- data/lib/pgbus/table_maintenance.rb +13 -2
- data/lib/pgbus/uniqueness.rb +11 -12
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source.rb +36 -4
- data/lib/pgbus/web/health_app.rb +102 -0
- data/lib/pgbus/web/health_server.rb +144 -0
- data/lib/pgbus/web/payload_filter.rb +3 -3
- data/lib/pgbus/web/streamer/instance.rb +58 -2
- data/lib/pgbus/web/streamer/listener.rb +69 -21
- data/lib/pgbus.rb +77 -0
- data/lib/tasks/pgbus_doctor.rake +12 -0
- metadata +19 -4
- data/app/models/pgbus/job_lock.rb +0 -98
- data/lib/generators/pgbus/templates/add_job_locks.rb.erb +0 -21
data/lib/pgbus/configuration.rb
CHANGED
|
@@ -30,7 +30,10 @@ module Pgbus
|
|
|
30
30
|
# advanced for longer than stall_threshold seconds (default 90).
|
|
31
31
|
# read_timeout caps how long a single PGMQ read can block (default 30s),
|
|
32
32
|
# so a dead socket raises instead of parking the loop forever.
|
|
33
|
-
|
|
33
|
+
# drain_timeout bounds the graceful-shutdown drain phase (default 30s): a
|
|
34
|
+
# job still running after this is abandoned to shutdown's own termination
|
|
35
|
+
# wait, so recycling/deploy never wedges on a permanently-stuck job.
|
|
36
|
+
attr_accessor :stall_threshold, :read_timeout, :drain_timeout
|
|
34
37
|
|
|
35
38
|
# Dispatcher settings
|
|
36
39
|
attr_accessor :dispatch_interval
|
|
@@ -50,7 +53,8 @@ module Pgbus
|
|
|
50
53
|
# Priority queues
|
|
51
54
|
attr_accessor :priority_levels, :default_priority
|
|
52
55
|
|
|
53
|
-
# Grouped reads (PGMQ v1.11.0+ FIFO grouping).
|
|
56
|
+
# Grouped reads (PGMQ v1.11.0+ FIFO grouping). EXPERIMENTAL — exempt from
|
|
57
|
+
# the 1.0 stability promise; the shape may change in a minor release.
|
|
54
58
|
# nil = disabled (default read_batch behavior).
|
|
55
59
|
# :fifo = use read_grouped (drains oldest group first, throughput-optimized).
|
|
56
60
|
# :round_robin = use read_grouped_rr (fair round-robin across groups).
|
|
@@ -86,10 +90,10 @@ module Pgbus
|
|
|
86
90
|
attr_reader :pgmq_schema_mode
|
|
87
91
|
|
|
88
92
|
# Event consumers
|
|
89
|
-
|
|
93
|
+
attr_reader :event_consumers
|
|
90
94
|
|
|
91
95
|
# Recurring jobs
|
|
92
|
-
attr_accessor :recurring_tasks, :recurring_schedule_interval, :
|
|
96
|
+
attr_accessor :recurring_tasks, :recurring_schedule_interval, :recurring_enabled, :recurring_tasks_file
|
|
93
97
|
attr_writer :recurring_tasks_files
|
|
94
98
|
attr_reader :recurring_execution_retention # rubocop:disable Style/AccessorGrouping
|
|
95
99
|
|
|
@@ -103,14 +107,21 @@ module Pgbus
|
|
|
103
107
|
attr_accessor :zombie_detection
|
|
104
108
|
|
|
105
109
|
# Job stats
|
|
106
|
-
attr_accessor :stats_enabled
|
|
110
|
+
attr_accessor :stats_enabled, :stats_flush_size, :stats_flush_interval
|
|
107
111
|
attr_reader :stats_retention # rubocop:disable Style/AccessorGrouping
|
|
108
112
|
|
|
109
113
|
# Web dashboard
|
|
110
114
|
attr_accessor :web_auth, :web_refresh_interval, :web_per_page, :web_live_updates, :web_data_source,
|
|
111
115
|
:insights_default_minutes, :base_controller_class, :return_to_app_url,
|
|
112
116
|
:metrics_enabled,
|
|
113
|
-
:
|
|
117
|
+
:web_filter_parameters, :web_filter_sensitive
|
|
118
|
+
|
|
119
|
+
# HTTP health endpoints (liveness/readiness for orchestrators like
|
|
120
|
+
# Kubernetes). `health_port` nil (default) leaves the standalone server in
|
|
121
|
+
# the supervisor disabled — mount Pgbus::Web::HealthApp in the host Rails
|
|
122
|
+
# app instead. When set, Supervisor#run serves /livez and /readyz over a
|
|
123
|
+
# plain TCPServer bound to `health_bind` (default localhost).
|
|
124
|
+
attr_accessor :health_port, :health_bind
|
|
114
125
|
|
|
115
126
|
# Streams (turbo-rails replacement, SSE-based)
|
|
116
127
|
attr_accessor :streams_enabled, :streams_path, :streams_queue_prefix, :streams_signed_name_secret,
|
|
@@ -136,6 +147,21 @@ module Pgbus
|
|
|
136
147
|
# Set to false to opt out without uninstalling the appsignal gem.
|
|
137
148
|
attr_accessor :appsignal_enabled, :appsignal_probe_enabled
|
|
138
149
|
|
|
150
|
+
# Generic metrics adapter (Pgbus::Metrics). Consumes the same pgbus.*
|
|
151
|
+
# instrumentation events as AppSignal and forwards them to a backend:
|
|
152
|
+
# nil (default) — install nothing, zero overhead
|
|
153
|
+
# :prometheus — in-process registry, scraped via PrometheusExporter
|
|
154
|
+
# :statsd — UDP datagrams to statsd_host:statsd_port
|
|
155
|
+
# a Pgbus::Metrics::Backend instance — a custom backend
|
|
156
|
+
attr_accessor :metrics_backend, :statsd_host, :statsd_port
|
|
157
|
+
|
|
158
|
+
# When true (default), Pgbus.configure and ConfigLoader.apply run
|
|
159
|
+
# validate! after applying settings, so an invalid value fails loud at
|
|
160
|
+
# boot instead of dormant until a worker path consumes it. Set false to
|
|
161
|
+
# opt out (exotic setups that intentionally hold a transiently-invalid
|
|
162
|
+
# config); explicit validate! still works.
|
|
163
|
+
attr_accessor :eager_validation
|
|
164
|
+
|
|
139
165
|
def initialize
|
|
140
166
|
@database_url = nil
|
|
141
167
|
@connection_params = nil
|
|
@@ -159,6 +185,7 @@ module Pgbus
|
|
|
159
185
|
|
|
160
186
|
@stall_threshold = 90
|
|
161
187
|
@read_timeout = 30
|
|
188
|
+
@drain_timeout = 30
|
|
162
189
|
|
|
163
190
|
@dispatch_interval = 1.0
|
|
164
191
|
|
|
@@ -202,13 +229,18 @@ module Pgbus
|
|
|
202
229
|
@recurring_schedule_interval = 1.0
|
|
203
230
|
@recurring_tasks_file = nil
|
|
204
231
|
@recurring_tasks_files = nil
|
|
205
|
-
@
|
|
232
|
+
@recurring_enabled = true
|
|
206
233
|
@recurring_execution_retention = 7 * 24 * 3600 # 7 days
|
|
207
234
|
|
|
208
235
|
@zombie_detection = true
|
|
209
236
|
|
|
210
237
|
@stats_enabled = true
|
|
211
238
|
@stats_retention = 30 * 24 * 3600 # 30 days
|
|
239
|
+
# StatBuffer flush thresholds. Buffered stats are lost on SIGKILL, so a
|
|
240
|
+
# smaller size/interval shrinks the residual loss window at the cost of
|
|
241
|
+
# more frequent bulk inserts. Defaults mirror StatBuffer's own constants.
|
|
242
|
+
@stats_flush_size = StatBuffer::DEFAULT_FLUSH_SIZE
|
|
243
|
+
@stats_flush_interval = StatBuffer::DEFAULT_FLUSH_INTERVAL
|
|
212
244
|
|
|
213
245
|
@connects_to = nil
|
|
214
246
|
|
|
@@ -221,8 +253,13 @@ module Pgbus
|
|
|
221
253
|
@base_controller_class = "::ActionController::Base"
|
|
222
254
|
@return_to_app_url = nil
|
|
223
255
|
@metrics_enabled = true
|
|
224
|
-
@
|
|
225
|
-
@
|
|
256
|
+
@web_filter_parameters = nil # nil = auto-detect from Rails, then fall back to defaults
|
|
257
|
+
@web_filter_sensitive = true
|
|
258
|
+
@deprecation_warned = Set.new
|
|
259
|
+
|
|
260
|
+
# HTTP health endpoints — nil port disables the standalone server.
|
|
261
|
+
@health_port = nil
|
|
262
|
+
@health_bind = "127.0.0.1"
|
|
226
263
|
|
|
227
264
|
@streams_enabled = true
|
|
228
265
|
@streams_path = nil
|
|
@@ -259,6 +296,7 @@ module Pgbus
|
|
|
259
296
|
# PG keepalive interval.
|
|
260
297
|
@streams_listen_health_check_ms = 250
|
|
261
298
|
@streams_write_deadline_ms = 5_000
|
|
299
|
+
# EXPERIMENTAL — exempt from the 1.0 stability promise.
|
|
262
300
|
@streams_falcon_streaming_body = false
|
|
263
301
|
# Opt-in: when true, the Dispatcher writes one row to
|
|
264
302
|
# pgbus_stream_stats per broadcast/connect/disconnect. Default
|
|
@@ -273,7 +311,8 @@ module Pgbus
|
|
|
273
311
|
@streams_orphan_sweep_interval = 3600 # 1 hour
|
|
274
312
|
@streams_orphan_threshold = 86_400 # 24 hours
|
|
275
313
|
@streams_durable_patterns = []
|
|
276
|
-
#
|
|
314
|
+
# EXPERIMENTAL (both presence settings) — exempt from the 1.0 stability
|
|
315
|
+
# promise. Streams matching these patterns get connection-driven presence:
|
|
277
316
|
# auto-join on SSE connect, auto-leave on disconnect, touch on the
|
|
278
317
|
# keepalive heartbeat (issue #169). Empty by default (opt-in).
|
|
279
318
|
@streams_presence_patterns = []
|
|
@@ -287,6 +326,13 @@ module Pgbus
|
|
|
287
326
|
# the same process, so the operator can disable it independently.
|
|
288
327
|
@appsignal_enabled = true
|
|
289
328
|
@appsignal_probe_enabled = true
|
|
329
|
+
|
|
330
|
+
# Generic metrics adapter: off by default (no subscription installed).
|
|
331
|
+
@metrics_backend = nil
|
|
332
|
+
@statsd_host = "127.0.0.1"
|
|
333
|
+
@statsd_port = 8125
|
|
334
|
+
|
|
335
|
+
@eager_validation = true
|
|
290
336
|
end
|
|
291
337
|
|
|
292
338
|
def queue_name(name)
|
|
@@ -311,7 +357,7 @@ module Pgbus
|
|
|
311
357
|
# Returns the execution mode for a specific worker config hash,
|
|
312
358
|
# falling back to the global execution_mode setting.
|
|
313
359
|
def execution_mode_for(worker_config)
|
|
314
|
-
mode = worker_config[:execution_mode] ||
|
|
360
|
+
mode = worker_config[:execution_mode] || execution_mode
|
|
315
361
|
ExecutionPools.normalize_mode(mode)
|
|
316
362
|
end
|
|
317
363
|
|
|
@@ -320,10 +366,17 @@ module Pgbus
|
|
|
320
366
|
def log_format=(format)
|
|
321
367
|
format = format.to_sym
|
|
322
368
|
unless VALID_LOG_FORMATS.include?(format)
|
|
323
|
-
raise
|
|
369
|
+
raise Pgbus::ConfigurationError, "Invalid log_format: #{format}. Must be one of: #{VALID_LOG_FORMATS.join(", ")}"
|
|
324
370
|
end
|
|
325
371
|
|
|
326
372
|
@log_format = format
|
|
373
|
+
|
|
374
|
+
# Only install pgbus's formatter when the logger still carries a nil or
|
|
375
|
+
# pgbus-installed formatter. A user who set a custom logger with a custom
|
|
376
|
+
# formatter keeps it — log_format only records the intended format for
|
|
377
|
+
# pgbus's own defaults, it must not silently clobber their formatting.
|
|
378
|
+
return unless pgbus_installable_formatter?(@logger&.formatter)
|
|
379
|
+
|
|
327
380
|
@logger.formatter = case format
|
|
328
381
|
when :json then LogFormatter::JSON.new
|
|
329
382
|
when :text then LogFormatter::Text.new
|
|
@@ -335,7 +388,7 @@ module Pgbus
|
|
|
335
388
|
def streams_default_broadcast_mode=(mode)
|
|
336
389
|
mode = mode.to_sym
|
|
337
390
|
unless VALID_BROADCAST_MODES.include?(mode)
|
|
338
|
-
raise
|
|
391
|
+
raise Pgbus::ConfigurationError,
|
|
339
392
|
"Invalid streams_default_broadcast_mode: #{mode}. Must be one of: #{VALID_BROADCAST_MODES.join(", ")}"
|
|
340
393
|
end
|
|
341
394
|
|
|
@@ -350,11 +403,11 @@ module Pgbus
|
|
|
350
403
|
when Symbol then mode
|
|
351
404
|
when String then mode.to_sym
|
|
352
405
|
else
|
|
353
|
-
raise
|
|
406
|
+
raise Pgbus::ConfigurationError,
|
|
354
407
|
"Invalid group_mode type: #{mode.class}. Must be nil, String, or Symbol"
|
|
355
408
|
end
|
|
356
409
|
unless VALID_GROUP_MODES.include?(coerced)
|
|
357
|
-
raise
|
|
410
|
+
raise Pgbus::ConfigurationError, "Invalid group_mode: #{coerced.inspect}. Must be nil, :fifo, or :round_robin"
|
|
358
411
|
end
|
|
359
412
|
|
|
360
413
|
@group_mode = coerced
|
|
@@ -365,7 +418,7 @@ module Pgbus
|
|
|
365
418
|
def pgmq_schema_mode=(mode)
|
|
366
419
|
mode = mode.to_sym
|
|
367
420
|
unless VALID_PGMQ_SCHEMA_MODES.include?(mode)
|
|
368
|
-
raise
|
|
421
|
+
raise Pgbus::ConfigurationError, "Invalid pgmq_schema_mode: #{mode}. Must be one of: #{VALID_PGMQ_SCHEMA_MODES.join(", ")}"
|
|
369
422
|
end
|
|
370
423
|
|
|
371
424
|
@pgmq_schema_mode = mode
|
|
@@ -373,96 +426,143 @@ module Pgbus
|
|
|
373
426
|
|
|
374
427
|
def validate!
|
|
375
428
|
if pool_size && !(pool_size.is_a?(Numeric) && pool_size.positive?)
|
|
376
|
-
raise
|
|
429
|
+
raise Pgbus::ConfigurationError, "pool_size must be a positive number or nil (auto-tune)"
|
|
377
430
|
end
|
|
378
431
|
|
|
379
|
-
raise
|
|
380
|
-
raise
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
432
|
+
raise Pgbus::ConfigurationError, "pool_timeout must be > 0" unless pool_timeout.is_a?(Numeric) && pool_timeout.positive?
|
|
433
|
+
raise Pgbus::ConfigurationError, "polling_interval must be > 0" unless polling_interval.is_a?(Numeric) && polling_interval.positive?
|
|
434
|
+
unless visibility_timeout.is_a?(Numeric) && visibility_timeout.positive?
|
|
435
|
+
raise Pgbus::ConfigurationError,
|
|
436
|
+
"visibility_timeout must be > 0"
|
|
437
|
+
end
|
|
438
|
+
raise Pgbus::ConfigurationError, "max_retries must be >= 0" unless max_retries.is_a?(Integer) && max_retries >= 0
|
|
439
|
+
raise Pgbus::ConfigurationError, "retry_backoff must be > 0" unless retry_backoff.is_a?(Numeric) && retry_backoff.positive?
|
|
440
|
+
unless retry_backoff_max.is_a?(Numeric) && retry_backoff_max.positive?
|
|
441
|
+
raise Pgbus::ConfigurationError,
|
|
442
|
+
"retry_backoff_max must be > 0"
|
|
443
|
+
end
|
|
385
444
|
unless retry_backoff_jitter.is_a?(Numeric) && retry_backoff_jitter >= 0 && retry_backoff_jitter <= 1
|
|
386
|
-
raise
|
|
445
|
+
raise Pgbus::ConfigurationError, "retry_backoff_jitter must be between 0 and 1"
|
|
387
446
|
end
|
|
388
447
|
|
|
389
448
|
unless stall_threshold.nil? || (stall_threshold.is_a?(Numeric) && stall_threshold.positive?)
|
|
390
|
-
raise
|
|
449
|
+
raise Pgbus::ConfigurationError, "stall_threshold must be a positive number or nil to disable"
|
|
391
450
|
end
|
|
392
451
|
unless read_timeout.nil? || (read_timeout.is_a?(Numeric) && read_timeout.positive?)
|
|
393
|
-
raise
|
|
452
|
+
raise Pgbus::ConfigurationError, "read_timeout must be a positive number or nil to disable"
|
|
453
|
+
end
|
|
454
|
+
raise Pgbus::ConfigurationError, "drain_timeout must be > 0" unless drain_timeout.is_a?(Numeric) && drain_timeout.positive?
|
|
455
|
+
|
|
456
|
+
unless stats_flush_size.is_a?(Integer) && stats_flush_size.positive?
|
|
457
|
+
raise Pgbus::ConfigurationError, "stats_flush_size must be a positive integer"
|
|
458
|
+
end
|
|
459
|
+
unless stats_flush_interval.is_a?(Numeric) && stats_flush_interval.positive?
|
|
460
|
+
raise Pgbus::ConfigurationError, "stats_flush_interval must be a positive number"
|
|
461
|
+
end
|
|
462
|
+
|
|
463
|
+
unless health_port.nil? || (health_port.is_a?(Integer) && health_port.between?(1, 65_535))
|
|
464
|
+
raise Pgbus::ConfigurationError, "health_port must be an integer between 1 and 65535 or nil to disable"
|
|
465
|
+
end
|
|
466
|
+
|
|
467
|
+
raise Pgbus::ConfigurationError, "health_bind must be a non-empty String" unless health_bind.is_a?(String) && !health_bind.empty?
|
|
468
|
+
|
|
469
|
+
raise Pgbus::ConfigurationError, "statsd_host must be a non-empty String" unless statsd_host.is_a?(String) && !statsd_host.empty?
|
|
470
|
+
|
|
471
|
+
unless statsd_port.is_a?(Integer) && statsd_port.between?(1, 65_535)
|
|
472
|
+
raise Pgbus::ConfigurationError, "statsd_port must be an integer between 1 and 65535"
|
|
394
473
|
end
|
|
395
474
|
|
|
396
475
|
# Validate global execution_mode
|
|
397
|
-
|
|
476
|
+
validate_execution_mode!(execution_mode)
|
|
398
477
|
|
|
399
478
|
Array(workers).each do |w|
|
|
400
|
-
threads = w[:threads] ||
|
|
401
|
-
raise
|
|
479
|
+
threads = w[:threads] || 5
|
|
480
|
+
raise Pgbus::ConfigurationError, "worker threads must be > 0" unless threads.is_a?(Integer) && threads.positive?
|
|
402
481
|
|
|
403
482
|
# Validate per-worker execution_mode override if present
|
|
404
|
-
mode = w[:execution_mode]
|
|
405
|
-
|
|
483
|
+
mode = w[:execution_mode]
|
|
484
|
+
validate_execution_mode!(mode) if mode
|
|
406
485
|
end
|
|
407
486
|
|
|
408
|
-
|
|
487
|
+
if prefetch_limit && !(prefetch_limit.is_a?(Integer) && prefetch_limit.positive?)
|
|
488
|
+
raise Pgbus::ConfigurationError,
|
|
489
|
+
"prefetch_limit must be > 0"
|
|
490
|
+
end
|
|
409
491
|
|
|
410
492
|
if priority_levels && !(priority_levels.is_a?(Integer) && priority_levels >= 1 && priority_levels <= 10)
|
|
411
|
-
raise
|
|
493
|
+
raise Pgbus::ConfigurationError, "priority_levels must be an integer between 1 and 10"
|
|
412
494
|
end
|
|
413
495
|
|
|
414
496
|
unless insights_default_minutes.is_a?(Integer) && insights_default_minutes.positive?
|
|
415
|
-
raise
|
|
497
|
+
raise Pgbus::ConfigurationError, "insights_default_minutes must be a positive integer"
|
|
416
498
|
end
|
|
417
499
|
|
|
418
500
|
validate_streams!
|
|
501
|
+
validate_metrics_backend!
|
|
419
502
|
|
|
420
503
|
self
|
|
421
504
|
end
|
|
422
505
|
|
|
423
506
|
def validate_streams!
|
|
424
507
|
unless streams_default_retention.is_a?(Numeric) && streams_default_retention >= 0
|
|
425
|
-
raise
|
|
508
|
+
raise Pgbus::ConfigurationError, "streams_default_retention must be a non-negative number"
|
|
426
509
|
end
|
|
427
510
|
|
|
428
511
|
unless streams_max_connections.is_a?(Integer) && streams_max_connections.positive?
|
|
429
|
-
raise
|
|
512
|
+
raise Pgbus::ConfigurationError, "streams_max_connections must be a positive integer"
|
|
430
513
|
end
|
|
431
514
|
|
|
432
515
|
unless streams_heartbeat_interval.is_a?(Numeric) && streams_heartbeat_interval.positive?
|
|
433
|
-
raise
|
|
516
|
+
raise Pgbus::ConfigurationError, "streams_heartbeat_interval must be a positive number"
|
|
434
517
|
end
|
|
435
518
|
|
|
436
519
|
unless streams_idle_timeout.is_a?(Numeric) && streams_idle_timeout.positive?
|
|
437
|
-
raise
|
|
520
|
+
raise Pgbus::ConfigurationError, "streams_idle_timeout must be a positive number"
|
|
438
521
|
end
|
|
439
522
|
|
|
440
523
|
unless streams_listen_health_check_ms.is_a?(Integer) && streams_listen_health_check_ms.positive?
|
|
441
|
-
raise
|
|
524
|
+
raise Pgbus::ConfigurationError, "streams_listen_health_check_ms must be a positive integer"
|
|
442
525
|
end
|
|
443
526
|
|
|
444
527
|
unless streams_write_deadline_ms.is_a?(Integer) && streams_write_deadline_ms.positive?
|
|
445
|
-
raise
|
|
528
|
+
raise Pgbus::ConfigurationError, "streams_write_deadline_ms must be a positive integer"
|
|
446
529
|
end
|
|
447
530
|
|
|
448
|
-
raise
|
|
531
|
+
raise Pgbus::ConfigurationError, "streams_retention must be a Hash" unless streams_retention.is_a?(Hash)
|
|
449
532
|
|
|
450
533
|
if streams_orphan_sweep_interval && !(streams_orphan_sweep_interval.is_a?(Numeric) && streams_orphan_sweep_interval.positive?)
|
|
451
|
-
raise
|
|
534
|
+
raise Pgbus::ConfigurationError, "streams_orphan_sweep_interval must be a positive number or nil to disable"
|
|
452
535
|
end
|
|
453
536
|
|
|
454
|
-
|
|
537
|
+
unless streams_durable_patterns.is_a?(Array)
|
|
538
|
+
raise Pgbus::ConfigurationError,
|
|
539
|
+
"streams_durable_patterns must be an Array of strings/regex"
|
|
540
|
+
end
|
|
455
541
|
|
|
456
|
-
|
|
542
|
+
unless streams_presence_patterns.is_a?(Array)
|
|
543
|
+
raise Pgbus::ConfigurationError,
|
|
544
|
+
"streams_presence_patterns must be an Array of strings/regex"
|
|
545
|
+
end
|
|
457
546
|
|
|
458
547
|
if !streams_presence_member.nil? && !streams_presence_member.respond_to?(:call)
|
|
459
|
-
raise
|
|
548
|
+
raise Pgbus::ConfigurationError, "streams_presence_member must respond to #call (a Proc/lambda) or be nil"
|
|
460
549
|
end
|
|
461
550
|
|
|
462
551
|
return if streams_orphan_threshold.nil?
|
|
463
552
|
return if streams_orphan_threshold.is_a?(Numeric) && streams_orphan_threshold.positive?
|
|
464
553
|
|
|
465
|
-
raise
|
|
554
|
+
raise Pgbus::ConfigurationError, "streams_orphan_threshold must be a positive number or nil to disable"
|
|
555
|
+
end
|
|
556
|
+
|
|
557
|
+
def validate_metrics_backend!
|
|
558
|
+
value = metrics_backend
|
|
559
|
+
return if value.nil?
|
|
560
|
+
return if %i[prometheus statsd].include?(value)
|
|
561
|
+
return if defined?(Pgbus::Metrics::Backend) && value.is_a?(Pgbus::Metrics::Backend)
|
|
562
|
+
|
|
563
|
+
raise Pgbus::ConfigurationError,
|
|
564
|
+
"metrics_backend must be nil, :prometheus, :statsd, or a " \
|
|
565
|
+
"Pgbus::Metrics::Backend instance (got #{value.inspect})"
|
|
466
566
|
end
|
|
467
567
|
|
|
468
568
|
# Returns true if the given stream name should be durable based on
|
|
@@ -513,9 +613,11 @@ module Pgbus
|
|
|
513
613
|
# c.workers "*: 5"
|
|
514
614
|
# c.workers "critical: 5; default, mailers: 10"
|
|
515
615
|
#
|
|
516
|
-
# Array —
|
|
616
|
+
# Array — explicit form. Each entry is a Hash with :queues
|
|
517
617
|
# and :threads (and optionally :name, :single_active_consumer,
|
|
518
|
-
# :consumer_priority, :prefetch_limit).
|
|
618
|
+
# :consumer_priority, :prefetch_limit). This is the only way to
|
|
619
|
+
# express N identical anonymous capsules, so it is supported
|
|
620
|
+
# permanently alongside the String DSL.
|
|
519
621
|
#
|
|
520
622
|
# c.workers [{ queues: %w[default], threads: 5 }]
|
|
521
623
|
#
|
|
@@ -535,15 +637,15 @@ module Pgbus
|
|
|
535
637
|
# "*: 5" -> one anonymous capsule (wildcard never names)
|
|
536
638
|
# "*: 3; *: 3; *: 3" -> three anonymous capsules — legal,
|
|
537
639
|
# represents "3 forks all reading every
|
|
538
|
-
# queue",
|
|
539
|
-
# `
|
|
640
|
+
# queue", the same shape as the Array form
|
|
641
|
+
# `3 × {queues: ["*"], threads: 3}`
|
|
540
642
|
# "default: 5; default: 3" -> two anonymous capsules — same logic
|
|
541
643
|
#
|
|
542
|
-
# The point of the carve-out is the
|
|
543
|
-
#
|
|
544
|
-
#
|
|
545
|
-
#
|
|
546
|
-
#
|
|
644
|
+
# The point of the carve-out is the "I want N forks of the same worker
|
|
645
|
+
# pool" pattern: it must keep working since PGMQ tolerates it natively
|
|
646
|
+
# (multiple processes reading the same queue with FOR UPDATE SKIP LOCKED).
|
|
647
|
+
# The CLI's --capsule selector only matches NAMED capsules, so anonymous
|
|
648
|
+
# duplicates can't be ambiguously addressed.
|
|
547
649
|
def workers=(value)
|
|
548
650
|
@workers = case value
|
|
549
651
|
when nil
|
|
@@ -552,13 +654,29 @@ module Pgbus
|
|
|
552
654
|
parsed = CapsuleDSL.parse(value)
|
|
553
655
|
assign_auto_names(parsed)
|
|
554
656
|
when Array
|
|
555
|
-
value
|
|
657
|
+
value.map { |entry| normalize_entry(entry, group: "worker") }
|
|
556
658
|
else
|
|
557
|
-
raise
|
|
558
|
-
"workers must be a String (DSL), Array (
|
|
659
|
+
raise Pgbus::ConfigurationError,
|
|
660
|
+
"workers must be a String (DSL), Array (explicit form), or nil — got #{value.class}"
|
|
559
661
|
end
|
|
560
662
|
end
|
|
561
663
|
|
|
664
|
+
# Event consumer configs arrive with symbol keys (Ruby DSL) or string keys
|
|
665
|
+
# (YAML via ConfigLoader). Normalize once here so every read site uses
|
|
666
|
+
# symbol access only. nil passes through (no consumers configured).
|
|
667
|
+
def event_consumers=(value)
|
|
668
|
+
@event_consumers =
|
|
669
|
+
case value
|
|
670
|
+
when nil
|
|
671
|
+
nil
|
|
672
|
+
when Array
|
|
673
|
+
value.map { |entry| normalize_entry(entry, group: "event_consumer") }
|
|
674
|
+
else
|
|
675
|
+
raise Pgbus::ConfigurationError,
|
|
676
|
+
"event_consumers must be an Array or nil — got #{value.class}"
|
|
677
|
+
end
|
|
678
|
+
end
|
|
679
|
+
|
|
562
680
|
# Define a named capsule and append it to the workers list.
|
|
563
681
|
#
|
|
564
682
|
# c.capsule :critical, queues: %w[critical], threads: 5
|
|
@@ -569,13 +687,15 @@ module Pgbus
|
|
|
569
687
|
# +c.workers "..."+ followed by +c.capsule :name, ...+ appends the
|
|
570
688
|
# named capsule to the list parsed from the string.
|
|
571
689
|
def capsule(name, queues:, threads:, **)
|
|
572
|
-
raise
|
|
573
|
-
raise
|
|
690
|
+
raise Pgbus::ConfigurationError, "capsule queues must be a non-empty Array" unless queues.is_a?(Array) && queues.any?
|
|
691
|
+
raise Pgbus::ConfigurationError, "capsule threads must be a positive Integer" unless threads.is_a?(Integer) && threads.positive?
|
|
574
692
|
|
|
575
693
|
normalized_name = name.to_s
|
|
576
694
|
@workers ||= []
|
|
577
695
|
|
|
578
|
-
|
|
696
|
+
if @workers.any? { |c| capsule_name(c) == normalized_name }
|
|
697
|
+
raise Pgbus::ConfigurationError, "capsule #{name.inspect} is already defined"
|
|
698
|
+
end
|
|
579
699
|
|
|
580
700
|
validate_no_queue_overlap!(queues)
|
|
581
701
|
|
|
@@ -611,9 +731,9 @@ module Pgbus
|
|
|
611
731
|
# Array — list of roles to boot
|
|
612
732
|
#
|
|
613
733
|
# Each role is normalized to a downcased symbol and validated against
|
|
614
|
-
# VALID_ROLES. Unknown role names raise
|
|
615
|
-
# typos like `[:workres]` fail loud at boot rather than
|
|
616
|
-
# supervisor idling with no children.
|
|
734
|
+
# VALID_ROLES. Unknown role names raise Pgbus::ConfigurationError
|
|
735
|
+
# immediately so typos like `[:workres]` fail loud at boot rather than
|
|
736
|
+
# leaving the supervisor idling with no children.
|
|
617
737
|
def roles=(value)
|
|
618
738
|
if value.nil?
|
|
619
739
|
@roles = nil
|
|
@@ -623,7 +743,7 @@ module Pgbus
|
|
|
623
743
|
normalized = Array(value).map { |r| r.to_s.downcase.to_sym }.uniq
|
|
624
744
|
invalid = normalized - VALID_ROLES
|
|
625
745
|
if invalid.any?
|
|
626
|
-
raise
|
|
746
|
+
raise Pgbus::ConfigurationError,
|
|
627
747
|
"invalid role(s) #{invalid.inspect} — valid roles are: #{VALID_ROLES.join(", ")}"
|
|
628
748
|
end
|
|
629
749
|
|
|
@@ -662,10 +782,56 @@ module Pgbus
|
|
|
662
782
|
@recurring_execution_retention = coerce_duration!(value, :recurring_execution_retention)
|
|
663
783
|
end
|
|
664
784
|
|
|
785
|
+
# recurring_tasks_file (singular) is deprecated in favor of the plural
|
|
786
|
+
# recurring_tasks_files. The engine still writes the singular internally to
|
|
787
|
+
# record which default file was loaded, so it stays a plain accessor; the
|
|
788
|
+
# deprecation warning fires only when a user has set BOTH — the case where
|
|
789
|
+
# the singular is silently ignored — pointing them at the plural.
|
|
665
790
|
def recurring_tasks_files
|
|
666
|
-
|
|
791
|
+
if @recurring_tasks_files
|
|
792
|
+
if @recurring_tasks_file
|
|
793
|
+
warn_deprecated_config(
|
|
794
|
+
:recurring_tasks_file,
|
|
795
|
+
"recurring_tasks_files is set, so recurring_tasks_file is ignored — " \
|
|
796
|
+
"consolidate onto recurring_tasks_files (plural)"
|
|
797
|
+
)
|
|
798
|
+
end
|
|
799
|
+
return @recurring_tasks_files
|
|
800
|
+
end
|
|
801
|
+
|
|
802
|
+
@recurring_tasks_file ? [@recurring_tasks_file] : nil
|
|
803
|
+
end
|
|
804
|
+
|
|
805
|
+
# --- Deprecated config aliases (renamed at 1.0.0, removed at 2.0.0) ---
|
|
806
|
+
|
|
807
|
+
# skip_recurring was the only negative-polarity toggle among the *_enabled
|
|
808
|
+
# switches. It now aliases recurring_enabled with inverted polarity.
|
|
809
|
+
def skip_recurring # rubocop:disable Naming/PredicateMethod
|
|
810
|
+
!recurring_enabled
|
|
811
|
+
end
|
|
812
|
+
|
|
813
|
+
def skip_recurring=(value)
|
|
814
|
+
warn_deprecated_config(:skip_recurring, "use recurring_enabled (positive polarity) instead")
|
|
815
|
+
self.recurring_enabled = !value
|
|
816
|
+
end
|
|
817
|
+
|
|
818
|
+
# dashboard_filter_* unify onto the incumbent web_ prefix.
|
|
819
|
+
def dashboard_filter_parameters
|
|
820
|
+
web_filter_parameters
|
|
821
|
+
end
|
|
822
|
+
|
|
823
|
+
def dashboard_filter_parameters=(value)
|
|
824
|
+
warn_deprecated_config(:dashboard_filter_parameters, "use web_filter_parameters instead")
|
|
825
|
+
self.web_filter_parameters = value
|
|
826
|
+
end
|
|
827
|
+
|
|
828
|
+
def dashboard_filter_sensitive
|
|
829
|
+
web_filter_sensitive
|
|
830
|
+
end
|
|
667
831
|
|
|
668
|
-
|
|
832
|
+
def dashboard_filter_sensitive=(value)
|
|
833
|
+
warn_deprecated_config(:dashboard_filter_sensitive, "use web_filter_sensitive instead")
|
|
834
|
+
self.web_filter_sensitive = value
|
|
669
835
|
end
|
|
670
836
|
|
|
671
837
|
# Returns the connection pool size to use for the PGMQ client.
|
|
@@ -801,6 +967,36 @@ module Pgbus
|
|
|
801
967
|
|
|
802
968
|
private
|
|
803
969
|
|
|
970
|
+
# True when log_format= may install its own formatter: no formatter set yet
|
|
971
|
+
# (nil), a pgbus formatter we installed, or a framework DEFAULT formatter
|
|
972
|
+
# (Ruby's Logger::Formatter, Rails' SimpleFormatter) that the app didn't
|
|
973
|
+
# deliberately choose. A genuinely custom formatter — a Proc, or any other
|
|
974
|
+
# class — is a deliberate choice and is left untouched.
|
|
975
|
+
def pgbus_installable_formatter?(formatter)
|
|
976
|
+
return true if formatter.nil?
|
|
977
|
+
return true if formatter.is_a?(LogFormatter::Text) || formatter.is_a?(LogFormatter::JSON)
|
|
978
|
+
|
|
979
|
+
# instance_of? (not is_a?) so a user subclass of these still counts as custom.
|
|
980
|
+
return true if formatter.instance_of?(::Logger::Formatter)
|
|
981
|
+
return true if defined?(::ActiveSupport::Logger::SimpleFormatter) &&
|
|
982
|
+
formatter.instance_of?(::ActiveSupport::Logger::SimpleFormatter)
|
|
983
|
+
|
|
984
|
+
false
|
|
985
|
+
end
|
|
986
|
+
|
|
987
|
+
# Log a one-time deprecation warning for a renamed config key. Deduped per
|
|
988
|
+
# key on this Configuration instance so a setter called on every request
|
|
989
|
+
# (or a reader hit in a hot loop) warns once, not on every access.
|
|
990
|
+
def warn_deprecated_config(key, guidance)
|
|
991
|
+
@deprecation_warned ||= Set.new
|
|
992
|
+
return unless @deprecation_warned.add?(key)
|
|
993
|
+
|
|
994
|
+
Pgbus.logger.warn do
|
|
995
|
+
"[Pgbus] Configuration `#{key}` is deprecated and will be removed in 2.0 — #{guidance}. " \
|
|
996
|
+
"See https://pgbus.dev/docs/upgrading-pgbus"
|
|
997
|
+
end
|
|
998
|
+
end
|
|
999
|
+
|
|
804
1000
|
# Built-in presence-member extractor used when no custom
|
|
805
1001
|
# `streams_presence_member` is configured. Returns a { id:, metadata: }
|
|
806
1002
|
# Hash or nil; #presence_member_for normalizes the id/metadata.
|
|
@@ -840,21 +1036,40 @@ module Pgbus
|
|
|
840
1036
|
return validate_positive_duration!(value, name)
|
|
841
1037
|
end
|
|
842
1038
|
|
|
843
|
-
raise
|
|
1039
|
+
raise Pgbus::ConfigurationError,
|
|
844
1040
|
"#{name} must be a Numeric (seconds), ActiveSupport::Duration, or nil to disable, got #{value.inspect}"
|
|
845
1041
|
end
|
|
846
1042
|
|
|
847
1043
|
def validate_positive_duration!(numeric, name)
|
|
848
|
-
raise
|
|
1044
|
+
raise Pgbus::ConfigurationError, "#{name} must be a positive number, got #{numeric}" unless numeric.positive?
|
|
849
1045
|
|
|
850
1046
|
numeric
|
|
851
1047
|
end
|
|
852
1048
|
|
|
853
|
-
#
|
|
854
|
-
#
|
|
1049
|
+
# ExecutionPools.normalize_mode raises ArgumentError for an unknown mode —
|
|
1050
|
+
# that's the right shape for the factory's own callers, but a bad
|
|
1051
|
+
# execution_mode reaching here is a configuration failure, so re-raise it
|
|
1052
|
+
# as ConfigurationError to match the rest of validate! (issue #282).
|
|
1053
|
+
def validate_execution_mode!(mode)
|
|
1054
|
+
ExecutionPools.normalize_mode(mode)
|
|
1055
|
+
rescue ArgumentError => e
|
|
1056
|
+
raise Pgbus::ConfigurationError, e.message
|
|
1057
|
+
end
|
|
1058
|
+
|
|
1059
|
+
# Symbolize the top-level keys of a worker/consumer config entry so every
|
|
1060
|
+
# downstream read site can use symbol access only. YAML (via ConfigLoader)
|
|
1061
|
+
# yields string keys; the Ruby DSL and +capsule+ builder yield symbols.
|
|
1062
|
+
# Entries are flat hashes with array/scalar values — no deep recursion.
|
|
1063
|
+
def normalize_entry(entry, group:)
|
|
1064
|
+
raise Pgbus::ConfigurationError, "#{group} entry must be a Hash, got #{entry.class}: #{entry.inspect}" unless entry.is_a?(Hash)
|
|
1065
|
+
|
|
1066
|
+
entry.transform_keys(&:to_sym)
|
|
1067
|
+
end
|
|
1068
|
+
|
|
1069
|
+
# Read a capsule's name (symbol key after normalization), as a string for
|
|
1070
|
+
# comparison. Returns nil for unnamed (legacy) entries.
|
|
855
1071
|
def capsule_name(entry)
|
|
856
|
-
|
|
857
|
-
raw&.to_s
|
|
1072
|
+
entry[:name]&.to_s
|
|
858
1073
|
end
|
|
859
1074
|
|
|
860
1075
|
# Auto-assign :name to parsed capsules where the first queue token would
|
|
@@ -887,17 +1102,17 @@ module Pgbus
|
|
|
887
1102
|
existing_named = (@workers || []).select { |c| capsule_name(c) }
|
|
888
1103
|
return if existing_named.empty?
|
|
889
1104
|
|
|
890
|
-
existing_queues = existing_named.flat_map { |c| c[:queues] ||
|
|
1105
|
+
existing_queues = existing_named.flat_map { |c| c[:queues] || [] }
|
|
891
1106
|
return if existing_queues.empty?
|
|
892
1107
|
|
|
893
1108
|
if existing_queues.include?(CapsuleDSL::WILDCARD)
|
|
894
|
-
raise
|
|
1109
|
+
raise Pgbus::ConfigurationError,
|
|
895
1110
|
"an existing named capsule already uses '*' (matches every queue) — " \
|
|
896
1111
|
"the new capsule's queues #{new_queues.inspect} would overlap with it"
|
|
897
1112
|
end
|
|
898
1113
|
|
|
899
1114
|
if new_queues.include?(CapsuleDSL::WILDCARD)
|
|
900
|
-
raise
|
|
1115
|
+
raise Pgbus::ConfigurationError,
|
|
901
1116
|
"the new capsule uses '*' (matches every queue) but other named capsules " \
|
|
902
1117
|
"are already defined with queues #{existing_queues.inspect} — " \
|
|
903
1118
|
"the wildcard would overlap with all of them"
|
|
@@ -906,7 +1121,7 @@ module Pgbus
|
|
|
906
1121
|
conflict = new_queues.find { |q| existing_queues.include?(q) }
|
|
907
1122
|
return unless conflict
|
|
908
1123
|
|
|
909
|
-
raise
|
|
1124
|
+
raise Pgbus::ConfigurationError,
|
|
910
1125
|
"queue #{conflict.inspect} is already assigned to another named capsule — " \
|
|
911
1126
|
"named capsules cannot share queues"
|
|
912
1127
|
end
|
|
@@ -915,9 +1130,9 @@ module Pgbus
|
|
|
915
1130
|
return 0 unless entries
|
|
916
1131
|
|
|
917
1132
|
entries.sum do |entry|
|
|
918
|
-
threads = entry[:threads] ||
|
|
1133
|
+
threads = entry[:threads] || default_threads
|
|
919
1134
|
unless threads.is_a?(Integer) && threads.positive?
|
|
920
|
-
raise
|
|
1135
|
+
raise Pgbus::ConfigurationError,
|
|
921
1136
|
"#{group} threads must be a positive integer, got #{threads.inspect}"
|
|
922
1137
|
end
|
|
923
1138
|
|