pgbus 0.9.8 → 0.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +329 -0
- data/README.md +336 -25
- data/lib/generators/pgbus/{add_job_locks_generator.rb → add_uniqueness_keys_generator.rb} +5 -5
- data/lib/pgbus/active_job/adapter.rb +1 -1
- data/lib/pgbus/config_loader.rb +29 -2
- data/lib/pgbus/configuration.rb +202 -73
- data/lib/pgbus/doctor.rb +28 -3
- data/lib/pgbus/execution_pools/async_pool.rb +2 -2
- data/lib/pgbus/generators/config_converter.rb +7 -5
- data/lib/pgbus/generators/migration_detector.rb +20 -16
- data/lib/pgbus/instrumentation.rb +2 -1
- data/lib/pgbus/integrations/appsignal/subscriber.rb +17 -2
- data/lib/pgbus/metrics/subscriber.rb +26 -2
- data/lib/pgbus/metrics.rb +3 -2
- data/lib/pgbus/pgmq_schema/pgmq_v1.11.1.sql +2126 -0
- data/lib/pgbus/pgmq_schema.rb +7 -2
- data/lib/pgbus/process/consumer.rb +141 -12
- data/lib/pgbus/process/supervisor.rb +33 -10
- data/lib/pgbus/process/worker.rb +6 -16
- data/lib/pgbus/recurring/schedule.rb +1 -2
- data/lib/pgbus/serializer.rb +4 -4
- data/lib/pgbus/streams/broadcastable_override.rb +0 -8
- data/lib/pgbus/streams/signed_name.rb +2 -2
- data/lib/pgbus/uniqueness.rb +11 -12
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source.rb +20 -4
- data/lib/pgbus/web/payload_filter.rb +3 -3
- data/lib/pgbus/web/streamer/instance.rb +6 -4
- data/lib/pgbus/web/streamer/listener.rb +21 -36
- data/lib/pgbus.rb +46 -6
- metadata +4 -6
- data/app/models/pgbus/job_lock.rb +0 -98
- data/lib/generators/pgbus/templates/add_job_locks.rb.erb +0 -21
- data/lib/rubocop/cop/pgbus/no_ruby_timeout.rb +0 -42
- data/lib/rubocop/pgbus.rb +0 -5
data/lib/pgbus/configuration.rb
CHANGED
|
@@ -30,7 +30,10 @@ module Pgbus
|
|
|
30
30
|
# advanced for longer than stall_threshold seconds (default 90).
|
|
31
31
|
# read_timeout caps how long a single PGMQ read can block (default 30s),
|
|
32
32
|
# so a dead socket raises instead of parking the loop forever.
|
|
33
|
-
|
|
33
|
+
# drain_timeout bounds the graceful-shutdown drain phase (default 30s): a
|
|
34
|
+
# job still running after this is abandoned to shutdown's own termination
|
|
35
|
+
# wait, so recycling/deploy never wedges on a permanently-stuck job.
|
|
36
|
+
attr_accessor :stall_threshold, :read_timeout, :drain_timeout
|
|
34
37
|
|
|
35
38
|
# Dispatcher settings
|
|
36
39
|
attr_accessor :dispatch_interval
|
|
@@ -50,7 +53,8 @@ module Pgbus
|
|
|
50
53
|
# Priority queues
|
|
51
54
|
attr_accessor :priority_levels, :default_priority
|
|
52
55
|
|
|
53
|
-
# Grouped reads (PGMQ v1.11.0+ FIFO grouping).
|
|
56
|
+
# Grouped reads (PGMQ v1.11.0+ FIFO grouping). EXPERIMENTAL — exempt from
|
|
57
|
+
# the 1.0 stability promise; the shape may change in a minor release.
|
|
54
58
|
# nil = disabled (default read_batch behavior).
|
|
55
59
|
# :fifo = use read_grouped (drains oldest group first, throughput-optimized).
|
|
56
60
|
# :round_robin = use read_grouped_rr (fair round-robin across groups).
|
|
@@ -89,7 +93,7 @@ module Pgbus
|
|
|
89
93
|
attr_reader :event_consumers
|
|
90
94
|
|
|
91
95
|
# Recurring jobs
|
|
92
|
-
attr_accessor :recurring_tasks, :recurring_schedule_interval, :
|
|
96
|
+
attr_accessor :recurring_tasks, :recurring_schedule_interval, :recurring_enabled, :recurring_tasks_file
|
|
93
97
|
attr_writer :recurring_tasks_files
|
|
94
98
|
attr_reader :recurring_execution_retention # rubocop:disable Style/AccessorGrouping
|
|
95
99
|
|
|
@@ -110,7 +114,7 @@ module Pgbus
|
|
|
110
114
|
attr_accessor :web_auth, :web_refresh_interval, :web_per_page, :web_live_updates, :web_data_source,
|
|
111
115
|
:insights_default_minutes, :base_controller_class, :return_to_app_url,
|
|
112
116
|
:metrics_enabled,
|
|
113
|
-
:
|
|
117
|
+
:web_filter_parameters, :web_filter_sensitive
|
|
114
118
|
|
|
115
119
|
# HTTP health endpoints (liveness/readiness for orchestrators like
|
|
116
120
|
# Kubernetes). `health_port` nil (default) leaves the standalone server in
|
|
@@ -181,6 +185,7 @@ module Pgbus
|
|
|
181
185
|
|
|
182
186
|
@stall_threshold = 90
|
|
183
187
|
@read_timeout = 30
|
|
188
|
+
@drain_timeout = 30
|
|
184
189
|
|
|
185
190
|
@dispatch_interval = 1.0
|
|
186
191
|
|
|
@@ -224,7 +229,7 @@ module Pgbus
|
|
|
224
229
|
@recurring_schedule_interval = 1.0
|
|
225
230
|
@recurring_tasks_file = nil
|
|
226
231
|
@recurring_tasks_files = nil
|
|
227
|
-
@
|
|
232
|
+
@recurring_enabled = true
|
|
228
233
|
@recurring_execution_retention = 7 * 24 * 3600 # 7 days
|
|
229
234
|
|
|
230
235
|
@zombie_detection = true
|
|
@@ -248,8 +253,9 @@ module Pgbus
|
|
|
248
253
|
@base_controller_class = "::ActionController::Base"
|
|
249
254
|
@return_to_app_url = nil
|
|
250
255
|
@metrics_enabled = true
|
|
251
|
-
@
|
|
252
|
-
@
|
|
256
|
+
@web_filter_parameters = nil # nil = auto-detect from Rails, then fall back to defaults
|
|
257
|
+
@web_filter_sensitive = true
|
|
258
|
+
@deprecation_warned = Set.new
|
|
253
259
|
|
|
254
260
|
# HTTP health endpoints — nil port disables the standalone server.
|
|
255
261
|
@health_port = nil
|
|
@@ -290,6 +296,7 @@ module Pgbus
|
|
|
290
296
|
# PG keepalive interval.
|
|
291
297
|
@streams_listen_health_check_ms = 250
|
|
292
298
|
@streams_write_deadline_ms = 5_000
|
|
299
|
+
# EXPERIMENTAL — exempt from the 1.0 stability promise.
|
|
293
300
|
@streams_falcon_streaming_body = false
|
|
294
301
|
# Opt-in: when true, the Dispatcher writes one row to
|
|
295
302
|
# pgbus_stream_stats per broadcast/connect/disconnect. Default
|
|
@@ -304,7 +311,8 @@ module Pgbus
|
|
|
304
311
|
@streams_orphan_sweep_interval = 3600 # 1 hour
|
|
305
312
|
@streams_orphan_threshold = 86_400 # 24 hours
|
|
306
313
|
@streams_durable_patterns = []
|
|
307
|
-
#
|
|
314
|
+
# EXPERIMENTAL (both presence settings) — exempt from the 1.0 stability
|
|
315
|
+
# promise. Streams matching these patterns get connection-driven presence:
|
|
308
316
|
# auto-join on SSE connect, auto-leave on disconnect, touch on the
|
|
309
317
|
# keepalive heartbeat (issue #169). Empty by default (opt-in).
|
|
310
318
|
@streams_presence_patterns = []
|
|
@@ -358,10 +366,17 @@ module Pgbus
|
|
|
358
366
|
def log_format=(format)
|
|
359
367
|
format = format.to_sym
|
|
360
368
|
unless VALID_LOG_FORMATS.include?(format)
|
|
361
|
-
raise
|
|
369
|
+
raise Pgbus::ConfigurationError, "Invalid log_format: #{format}. Must be one of: #{VALID_LOG_FORMATS.join(", ")}"
|
|
362
370
|
end
|
|
363
371
|
|
|
364
372
|
@log_format = format
|
|
373
|
+
|
|
374
|
+
# Only install pgbus's formatter when the logger still carries a nil or
|
|
375
|
+
# pgbus-installed formatter. A user who set a custom logger with a custom
|
|
376
|
+
# formatter keeps it — log_format only records the intended format for
|
|
377
|
+
# pgbus's own defaults, it must not silently clobber their formatting.
|
|
378
|
+
return unless pgbus_installable_formatter?(@logger&.formatter)
|
|
379
|
+
|
|
365
380
|
@logger.formatter = case format
|
|
366
381
|
when :json then LogFormatter::JSON.new
|
|
367
382
|
when :text then LogFormatter::Text.new
|
|
@@ -373,7 +388,7 @@ module Pgbus
|
|
|
373
388
|
def streams_default_broadcast_mode=(mode)
|
|
374
389
|
mode = mode.to_sym
|
|
375
390
|
unless VALID_BROADCAST_MODES.include?(mode)
|
|
376
|
-
raise
|
|
391
|
+
raise Pgbus::ConfigurationError,
|
|
377
392
|
"Invalid streams_default_broadcast_mode: #{mode}. Must be one of: #{VALID_BROADCAST_MODES.join(", ")}"
|
|
378
393
|
end
|
|
379
394
|
|
|
@@ -388,11 +403,11 @@ module Pgbus
|
|
|
388
403
|
when Symbol then mode
|
|
389
404
|
when String then mode.to_sym
|
|
390
405
|
else
|
|
391
|
-
raise
|
|
406
|
+
raise Pgbus::ConfigurationError,
|
|
392
407
|
"Invalid group_mode type: #{mode.class}. Must be nil, String, or Symbol"
|
|
393
408
|
end
|
|
394
409
|
unless VALID_GROUP_MODES.include?(coerced)
|
|
395
|
-
raise
|
|
410
|
+
raise Pgbus::ConfigurationError, "Invalid group_mode: #{coerced.inspect}. Must be nil, :fifo, or :round_robin"
|
|
396
411
|
end
|
|
397
412
|
|
|
398
413
|
@group_mode = coerced
|
|
@@ -403,7 +418,7 @@ module Pgbus
|
|
|
403
418
|
def pgmq_schema_mode=(mode)
|
|
404
419
|
mode = mode.to_sym
|
|
405
420
|
unless VALID_PGMQ_SCHEMA_MODES.include?(mode)
|
|
406
|
-
raise
|
|
421
|
+
raise Pgbus::ConfigurationError, "Invalid pgmq_schema_mode: #{mode}. Must be one of: #{VALID_PGMQ_SCHEMA_MODES.join(", ")}"
|
|
407
422
|
end
|
|
408
423
|
|
|
409
424
|
@pgmq_schema_mode = mode
|
|
@@ -411,57 +426,75 @@ module Pgbus
|
|
|
411
426
|
|
|
412
427
|
def validate!
|
|
413
428
|
if pool_size && !(pool_size.is_a?(Numeric) && pool_size.positive?)
|
|
414
|
-
raise
|
|
429
|
+
raise Pgbus::ConfigurationError, "pool_size must be a positive number or nil (auto-tune)"
|
|
415
430
|
end
|
|
416
431
|
|
|
417
|
-
raise
|
|
418
|
-
raise
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
432
|
+
raise Pgbus::ConfigurationError, "pool_timeout must be > 0" unless pool_timeout.is_a?(Numeric) && pool_timeout.positive?
|
|
433
|
+
raise Pgbus::ConfigurationError, "polling_interval must be > 0" unless polling_interval.is_a?(Numeric) && polling_interval.positive?
|
|
434
|
+
unless visibility_timeout.is_a?(Numeric) && visibility_timeout.positive?
|
|
435
|
+
raise Pgbus::ConfigurationError,
|
|
436
|
+
"visibility_timeout must be > 0"
|
|
437
|
+
end
|
|
438
|
+
raise Pgbus::ConfigurationError, "max_retries must be >= 0" unless max_retries.is_a?(Integer) && max_retries >= 0
|
|
439
|
+
raise Pgbus::ConfigurationError, "retry_backoff must be > 0" unless retry_backoff.is_a?(Numeric) && retry_backoff.positive?
|
|
440
|
+
unless retry_backoff_max.is_a?(Numeric) && retry_backoff_max.positive?
|
|
441
|
+
raise Pgbus::ConfigurationError,
|
|
442
|
+
"retry_backoff_max must be > 0"
|
|
443
|
+
end
|
|
423
444
|
unless retry_backoff_jitter.is_a?(Numeric) && retry_backoff_jitter >= 0 && retry_backoff_jitter <= 1
|
|
424
|
-
raise
|
|
445
|
+
raise Pgbus::ConfigurationError, "retry_backoff_jitter must be between 0 and 1"
|
|
425
446
|
end
|
|
426
447
|
|
|
427
448
|
unless stall_threshold.nil? || (stall_threshold.is_a?(Numeric) && stall_threshold.positive?)
|
|
428
|
-
raise
|
|
449
|
+
raise Pgbus::ConfigurationError, "stall_threshold must be a positive number or nil to disable"
|
|
429
450
|
end
|
|
430
451
|
unless read_timeout.nil? || (read_timeout.is_a?(Numeric) && read_timeout.positive?)
|
|
431
|
-
raise
|
|
452
|
+
raise Pgbus::ConfigurationError, "read_timeout must be a positive number or nil to disable"
|
|
432
453
|
end
|
|
454
|
+
raise Pgbus::ConfigurationError, "drain_timeout must be > 0" unless drain_timeout.is_a?(Numeric) && drain_timeout.positive?
|
|
433
455
|
|
|
434
456
|
unless stats_flush_size.is_a?(Integer) && stats_flush_size.positive?
|
|
435
|
-
raise
|
|
457
|
+
raise Pgbus::ConfigurationError, "stats_flush_size must be a positive integer"
|
|
436
458
|
end
|
|
437
459
|
unless stats_flush_interval.is_a?(Numeric) && stats_flush_interval.positive?
|
|
438
|
-
raise
|
|
460
|
+
raise Pgbus::ConfigurationError, "stats_flush_interval must be a positive number"
|
|
439
461
|
end
|
|
440
462
|
|
|
441
463
|
unless health_port.nil? || (health_port.is_a?(Integer) && health_port.between?(1, 65_535))
|
|
442
|
-
raise
|
|
464
|
+
raise Pgbus::ConfigurationError, "health_port must be an integer between 1 and 65535 or nil to disable"
|
|
465
|
+
end
|
|
466
|
+
|
|
467
|
+
raise Pgbus::ConfigurationError, "health_bind must be a non-empty String" unless health_bind.is_a?(String) && !health_bind.empty?
|
|
468
|
+
|
|
469
|
+
raise Pgbus::ConfigurationError, "statsd_host must be a non-empty String" unless statsd_host.is_a?(String) && !statsd_host.empty?
|
|
470
|
+
|
|
471
|
+
unless statsd_port.is_a?(Integer) && statsd_port.between?(1, 65_535)
|
|
472
|
+
raise Pgbus::ConfigurationError, "statsd_port must be an integer between 1 and 65535"
|
|
443
473
|
end
|
|
444
474
|
|
|
445
475
|
# Validate global execution_mode
|
|
446
|
-
|
|
476
|
+
validate_execution_mode!(execution_mode)
|
|
447
477
|
|
|
448
478
|
Array(workers).each do |w|
|
|
449
479
|
threads = w[:threads] || 5
|
|
450
|
-
raise
|
|
480
|
+
raise Pgbus::ConfigurationError, "worker threads must be > 0" unless threads.is_a?(Integer) && threads.positive?
|
|
451
481
|
|
|
452
482
|
# Validate per-worker execution_mode override if present
|
|
453
483
|
mode = w[:execution_mode]
|
|
454
|
-
|
|
484
|
+
validate_execution_mode!(mode) if mode
|
|
455
485
|
end
|
|
456
486
|
|
|
457
|
-
|
|
487
|
+
if prefetch_limit && !(prefetch_limit.is_a?(Integer) && prefetch_limit.positive?)
|
|
488
|
+
raise Pgbus::ConfigurationError,
|
|
489
|
+
"prefetch_limit must be > 0"
|
|
490
|
+
end
|
|
458
491
|
|
|
459
492
|
if priority_levels && !(priority_levels.is_a?(Integer) && priority_levels >= 1 && priority_levels <= 10)
|
|
460
|
-
raise
|
|
493
|
+
raise Pgbus::ConfigurationError, "priority_levels must be an integer between 1 and 10"
|
|
461
494
|
end
|
|
462
495
|
|
|
463
496
|
unless insights_default_minutes.is_a?(Integer) && insights_default_minutes.positive?
|
|
464
|
-
raise
|
|
497
|
+
raise Pgbus::ConfigurationError, "insights_default_minutes must be a positive integer"
|
|
465
498
|
end
|
|
466
499
|
|
|
467
500
|
validate_streams!
|
|
@@ -472,47 +505,53 @@ module Pgbus
|
|
|
472
505
|
|
|
473
506
|
def validate_streams!
|
|
474
507
|
unless streams_default_retention.is_a?(Numeric) && streams_default_retention >= 0
|
|
475
|
-
raise
|
|
508
|
+
raise Pgbus::ConfigurationError, "streams_default_retention must be a non-negative number"
|
|
476
509
|
end
|
|
477
510
|
|
|
478
511
|
unless streams_max_connections.is_a?(Integer) && streams_max_connections.positive?
|
|
479
|
-
raise
|
|
512
|
+
raise Pgbus::ConfigurationError, "streams_max_connections must be a positive integer"
|
|
480
513
|
end
|
|
481
514
|
|
|
482
515
|
unless streams_heartbeat_interval.is_a?(Numeric) && streams_heartbeat_interval.positive?
|
|
483
|
-
raise
|
|
516
|
+
raise Pgbus::ConfigurationError, "streams_heartbeat_interval must be a positive number"
|
|
484
517
|
end
|
|
485
518
|
|
|
486
519
|
unless streams_idle_timeout.is_a?(Numeric) && streams_idle_timeout.positive?
|
|
487
|
-
raise
|
|
520
|
+
raise Pgbus::ConfigurationError, "streams_idle_timeout must be a positive number"
|
|
488
521
|
end
|
|
489
522
|
|
|
490
523
|
unless streams_listen_health_check_ms.is_a?(Integer) && streams_listen_health_check_ms.positive?
|
|
491
|
-
raise
|
|
524
|
+
raise Pgbus::ConfigurationError, "streams_listen_health_check_ms must be a positive integer"
|
|
492
525
|
end
|
|
493
526
|
|
|
494
527
|
unless streams_write_deadline_ms.is_a?(Integer) && streams_write_deadline_ms.positive?
|
|
495
|
-
raise
|
|
528
|
+
raise Pgbus::ConfigurationError, "streams_write_deadline_ms must be a positive integer"
|
|
496
529
|
end
|
|
497
530
|
|
|
498
|
-
raise
|
|
531
|
+
raise Pgbus::ConfigurationError, "streams_retention must be a Hash" unless streams_retention.is_a?(Hash)
|
|
499
532
|
|
|
500
533
|
if streams_orphan_sweep_interval && !(streams_orphan_sweep_interval.is_a?(Numeric) && streams_orphan_sweep_interval.positive?)
|
|
501
|
-
raise
|
|
534
|
+
raise Pgbus::ConfigurationError, "streams_orphan_sweep_interval must be a positive number or nil to disable"
|
|
502
535
|
end
|
|
503
536
|
|
|
504
|
-
|
|
537
|
+
unless streams_durable_patterns.is_a?(Array)
|
|
538
|
+
raise Pgbus::ConfigurationError,
|
|
539
|
+
"streams_durable_patterns must be an Array of strings/regex"
|
|
540
|
+
end
|
|
505
541
|
|
|
506
|
-
|
|
542
|
+
unless streams_presence_patterns.is_a?(Array)
|
|
543
|
+
raise Pgbus::ConfigurationError,
|
|
544
|
+
"streams_presence_patterns must be an Array of strings/regex"
|
|
545
|
+
end
|
|
507
546
|
|
|
508
547
|
if !streams_presence_member.nil? && !streams_presence_member.respond_to?(:call)
|
|
509
|
-
raise
|
|
548
|
+
raise Pgbus::ConfigurationError, "streams_presence_member must respond to #call (a Proc/lambda) or be nil"
|
|
510
549
|
end
|
|
511
550
|
|
|
512
551
|
return if streams_orphan_threshold.nil?
|
|
513
552
|
return if streams_orphan_threshold.is_a?(Numeric) && streams_orphan_threshold.positive?
|
|
514
553
|
|
|
515
|
-
raise
|
|
554
|
+
raise Pgbus::ConfigurationError, "streams_orphan_threshold must be a positive number or nil to disable"
|
|
516
555
|
end
|
|
517
556
|
|
|
518
557
|
def validate_metrics_backend!
|
|
@@ -521,7 +560,7 @@ module Pgbus
|
|
|
521
560
|
return if %i[prometheus statsd].include?(value)
|
|
522
561
|
return if defined?(Pgbus::Metrics::Backend) && value.is_a?(Pgbus::Metrics::Backend)
|
|
523
562
|
|
|
524
|
-
raise
|
|
563
|
+
raise Pgbus::ConfigurationError,
|
|
525
564
|
"metrics_backend must be nil, :prometheus, :statsd, or a " \
|
|
526
565
|
"Pgbus::Metrics::Backend instance (got #{value.inspect})"
|
|
527
566
|
end
|
|
@@ -574,9 +613,11 @@ module Pgbus
|
|
|
574
613
|
# c.workers "*: 5"
|
|
575
614
|
# c.workers "critical: 5; default, mailers: 10"
|
|
576
615
|
#
|
|
577
|
-
# Array —
|
|
616
|
+
# Array — explicit form. Each entry is a Hash with :queues
|
|
578
617
|
# and :threads (and optionally :name, :single_active_consumer,
|
|
579
|
-
# :consumer_priority, :prefetch_limit).
|
|
618
|
+
# :consumer_priority, :prefetch_limit). This is the only way to
|
|
619
|
+
# express N identical anonymous capsules, so it is supported
|
|
620
|
+
# permanently alongside the String DSL.
|
|
580
621
|
#
|
|
581
622
|
# c.workers [{ queues: %w[default], threads: 5 }]
|
|
582
623
|
#
|
|
@@ -596,15 +637,15 @@ module Pgbus
|
|
|
596
637
|
# "*: 5" -> one anonymous capsule (wildcard never names)
|
|
597
638
|
# "*: 3; *: 3; *: 3" -> three anonymous capsules — legal,
|
|
598
639
|
# represents "3 forks all reading every
|
|
599
|
-
# queue",
|
|
600
|
-
# `
|
|
640
|
+
# queue", the same shape as the Array form
|
|
641
|
+
# `3 × {queues: ["*"], threads: 3}`
|
|
601
642
|
# "default: 5; default: 3" -> two anonymous capsules — same logic
|
|
602
643
|
#
|
|
603
|
-
# The point of the carve-out is the
|
|
604
|
-
#
|
|
605
|
-
#
|
|
606
|
-
#
|
|
607
|
-
#
|
|
644
|
+
# The point of the carve-out is the "I want N forks of the same worker
|
|
645
|
+
# pool" pattern: it must keep working since PGMQ tolerates it natively
|
|
646
|
+
# (multiple processes reading the same queue with FOR UPDATE SKIP LOCKED).
|
|
647
|
+
# The CLI's --capsule selector only matches NAMED capsules, so anonymous
|
|
648
|
+
# duplicates can't be ambiguously addressed.
|
|
608
649
|
def workers=(value)
|
|
609
650
|
@workers = case value
|
|
610
651
|
when nil
|
|
@@ -615,8 +656,8 @@ module Pgbus
|
|
|
615
656
|
when Array
|
|
616
657
|
value.map { |entry| normalize_entry(entry, group: "worker") }
|
|
617
658
|
else
|
|
618
|
-
raise
|
|
619
|
-
"workers must be a String (DSL), Array (
|
|
659
|
+
raise Pgbus::ConfigurationError,
|
|
660
|
+
"workers must be a String (DSL), Array (explicit form), or nil — got #{value.class}"
|
|
620
661
|
end
|
|
621
662
|
end
|
|
622
663
|
|
|
@@ -631,7 +672,7 @@ module Pgbus
|
|
|
631
672
|
when Array
|
|
632
673
|
value.map { |entry| normalize_entry(entry, group: "event_consumer") }
|
|
633
674
|
else
|
|
634
|
-
raise
|
|
675
|
+
raise Pgbus::ConfigurationError,
|
|
635
676
|
"event_consumers must be an Array or nil — got #{value.class}"
|
|
636
677
|
end
|
|
637
678
|
end
|
|
@@ -646,13 +687,15 @@ module Pgbus
|
|
|
646
687
|
# +c.workers "..."+ followed by +c.capsule :name, ...+ appends the
|
|
647
688
|
# named capsule to the list parsed from the string.
|
|
648
689
|
def capsule(name, queues:, threads:, **)
|
|
649
|
-
raise
|
|
650
|
-
raise
|
|
690
|
+
raise Pgbus::ConfigurationError, "capsule queues must be a non-empty Array" unless queues.is_a?(Array) && queues.any?
|
|
691
|
+
raise Pgbus::ConfigurationError, "capsule threads must be a positive Integer" unless threads.is_a?(Integer) && threads.positive?
|
|
651
692
|
|
|
652
693
|
normalized_name = name.to_s
|
|
653
694
|
@workers ||= []
|
|
654
695
|
|
|
655
|
-
|
|
696
|
+
if @workers.any? { |c| capsule_name(c) == normalized_name }
|
|
697
|
+
raise Pgbus::ConfigurationError, "capsule #{name.inspect} is already defined"
|
|
698
|
+
end
|
|
656
699
|
|
|
657
700
|
validate_no_queue_overlap!(queues)
|
|
658
701
|
|
|
@@ -688,9 +731,9 @@ module Pgbus
|
|
|
688
731
|
# Array — list of roles to boot
|
|
689
732
|
#
|
|
690
733
|
# Each role is normalized to a downcased symbol and validated against
|
|
691
|
-
# VALID_ROLES. Unknown role names raise
|
|
692
|
-
# typos like `[:workres]` fail loud at boot rather than
|
|
693
|
-
# supervisor idling with no children.
|
|
734
|
+
# VALID_ROLES. Unknown role names raise Pgbus::ConfigurationError
|
|
735
|
+
# immediately so typos like `[:workres]` fail loud at boot rather than
|
|
736
|
+
# leaving the supervisor idling with no children.
|
|
694
737
|
def roles=(value)
|
|
695
738
|
if value.nil?
|
|
696
739
|
@roles = nil
|
|
@@ -700,7 +743,7 @@ module Pgbus
|
|
|
700
743
|
normalized = Array(value).map { |r| r.to_s.downcase.to_sym }.uniq
|
|
701
744
|
invalid = normalized - VALID_ROLES
|
|
702
745
|
if invalid.any?
|
|
703
|
-
raise
|
|
746
|
+
raise Pgbus::ConfigurationError,
|
|
704
747
|
"invalid role(s) #{invalid.inspect} — valid roles are: #{VALID_ROLES.join(", ")}"
|
|
705
748
|
end
|
|
706
749
|
|
|
@@ -739,10 +782,56 @@ module Pgbus
|
|
|
739
782
|
@recurring_execution_retention = coerce_duration!(value, :recurring_execution_retention)
|
|
740
783
|
end
|
|
741
784
|
|
|
785
|
+
# recurring_tasks_file (singular) is deprecated in favor of the plural
|
|
786
|
+
# recurring_tasks_files. The engine still writes the singular internally to
|
|
787
|
+
# record which default file was loaded, so it stays a plain accessor; the
|
|
788
|
+
# deprecation warning fires only when a user has set BOTH — the case where
|
|
789
|
+
# the singular is silently ignored — pointing them at the plural.
|
|
742
790
|
def recurring_tasks_files
|
|
743
|
-
|
|
791
|
+
if @recurring_tasks_files
|
|
792
|
+
if @recurring_tasks_file
|
|
793
|
+
warn_deprecated_config(
|
|
794
|
+
:recurring_tasks_file,
|
|
795
|
+
"recurring_tasks_files is set, so recurring_tasks_file is ignored — " \
|
|
796
|
+
"consolidate onto recurring_tasks_files (plural)"
|
|
797
|
+
)
|
|
798
|
+
end
|
|
799
|
+
return @recurring_tasks_files
|
|
800
|
+
end
|
|
801
|
+
|
|
802
|
+
@recurring_tasks_file ? [@recurring_tasks_file] : nil
|
|
803
|
+
end
|
|
804
|
+
|
|
805
|
+
# --- Deprecated config aliases (renamed at 1.0.0, removed at 2.0.0) ---
|
|
806
|
+
|
|
807
|
+
# skip_recurring was the only negative-polarity toggle among the *_enabled
|
|
808
|
+
# switches. It now aliases recurring_enabled with inverted polarity.
|
|
809
|
+
def skip_recurring # rubocop:disable Naming/PredicateMethod
|
|
810
|
+
!recurring_enabled
|
|
811
|
+
end
|
|
744
812
|
|
|
745
|
-
|
|
813
|
+
def skip_recurring=(value)
|
|
814
|
+
warn_deprecated_config(:skip_recurring, "use recurring_enabled (positive polarity) instead")
|
|
815
|
+
self.recurring_enabled = !value
|
|
816
|
+
end
|
|
817
|
+
|
|
818
|
+
# dashboard_filter_* unify onto the incumbent web_ prefix.
|
|
819
|
+
def dashboard_filter_parameters
|
|
820
|
+
web_filter_parameters
|
|
821
|
+
end
|
|
822
|
+
|
|
823
|
+
def dashboard_filter_parameters=(value)
|
|
824
|
+
warn_deprecated_config(:dashboard_filter_parameters, "use web_filter_parameters instead")
|
|
825
|
+
self.web_filter_parameters = value
|
|
826
|
+
end
|
|
827
|
+
|
|
828
|
+
def dashboard_filter_sensitive
|
|
829
|
+
web_filter_sensitive
|
|
830
|
+
end
|
|
831
|
+
|
|
832
|
+
def dashboard_filter_sensitive=(value)
|
|
833
|
+
warn_deprecated_config(:dashboard_filter_sensitive, "use web_filter_sensitive instead")
|
|
834
|
+
self.web_filter_sensitive = value
|
|
746
835
|
end
|
|
747
836
|
|
|
748
837
|
# Returns the connection pool size to use for the PGMQ client.
|
|
@@ -878,6 +967,36 @@ module Pgbus
|
|
|
878
967
|
|
|
879
968
|
private
|
|
880
969
|
|
|
970
|
+
# True when log_format= may install its own formatter: no formatter set yet
|
|
971
|
+
# (nil), a pgbus formatter we installed, or a framework DEFAULT formatter
|
|
972
|
+
# (Ruby's Logger::Formatter, Rails' SimpleFormatter) that the app didn't
|
|
973
|
+
# deliberately choose. A genuinely custom formatter — a Proc, or any other
|
|
974
|
+
# class — is a deliberate choice and is left untouched.
|
|
975
|
+
def pgbus_installable_formatter?(formatter)
|
|
976
|
+
return true if formatter.nil?
|
|
977
|
+
return true if formatter.is_a?(LogFormatter::Text) || formatter.is_a?(LogFormatter::JSON)
|
|
978
|
+
|
|
979
|
+
# instance_of? (not is_a?) so a user subclass of these still counts as custom.
|
|
980
|
+
return true if formatter.instance_of?(::Logger::Formatter)
|
|
981
|
+
return true if defined?(::ActiveSupport::Logger::SimpleFormatter) &&
|
|
982
|
+
formatter.instance_of?(::ActiveSupport::Logger::SimpleFormatter)
|
|
983
|
+
|
|
984
|
+
false
|
|
985
|
+
end
|
|
986
|
+
|
|
987
|
+
# Log a one-time deprecation warning for a renamed config key. Deduped per
|
|
988
|
+
# key on this Configuration instance so a setter called on every request
|
|
989
|
+
# (or a reader hit in a hot loop) warns once, not on every access.
|
|
990
|
+
def warn_deprecated_config(key, guidance)
|
|
991
|
+
@deprecation_warned ||= Set.new
|
|
992
|
+
return unless @deprecation_warned.add?(key)
|
|
993
|
+
|
|
994
|
+
Pgbus.logger.warn do
|
|
995
|
+
"[Pgbus] Configuration `#{key}` is deprecated and will be removed in 2.0 — #{guidance}. " \
|
|
996
|
+
"See https://pgbus.dev/docs/upgrading-pgbus"
|
|
997
|
+
end
|
|
998
|
+
end
|
|
999
|
+
|
|
881
1000
|
# Built-in presence-member extractor used when no custom
|
|
882
1001
|
# `streams_presence_member` is configured. Returns a { id:, metadata: }
|
|
883
1002
|
# Hash or nil; #presence_member_for normalizes the id/metadata.
|
|
@@ -917,22 +1036,32 @@ module Pgbus
|
|
|
917
1036
|
return validate_positive_duration!(value, name)
|
|
918
1037
|
end
|
|
919
1038
|
|
|
920
|
-
raise
|
|
1039
|
+
raise Pgbus::ConfigurationError,
|
|
921
1040
|
"#{name} must be a Numeric (seconds), ActiveSupport::Duration, or nil to disable, got #{value.inspect}"
|
|
922
1041
|
end
|
|
923
1042
|
|
|
924
1043
|
def validate_positive_duration!(numeric, name)
|
|
925
|
-
raise
|
|
1044
|
+
raise Pgbus::ConfigurationError, "#{name} must be a positive number, got #{numeric}" unless numeric.positive?
|
|
926
1045
|
|
|
927
1046
|
numeric
|
|
928
1047
|
end
|
|
929
1048
|
|
|
1049
|
+
# ExecutionPools.normalize_mode raises ArgumentError for an unknown mode —
|
|
1050
|
+
# that's the right shape for the factory's own callers, but a bad
|
|
1051
|
+
# execution_mode reaching here is a configuration failure, so re-raise it
|
|
1052
|
+
# as ConfigurationError to match the rest of validate! (issue #282).
|
|
1053
|
+
def validate_execution_mode!(mode)
|
|
1054
|
+
ExecutionPools.normalize_mode(mode)
|
|
1055
|
+
rescue ArgumentError => e
|
|
1056
|
+
raise Pgbus::ConfigurationError, e.message
|
|
1057
|
+
end
|
|
1058
|
+
|
|
930
1059
|
# Symbolize the top-level keys of a worker/consumer config entry so every
|
|
931
1060
|
# downstream read site can use symbol access only. YAML (via ConfigLoader)
|
|
932
1061
|
# yields string keys; the Ruby DSL and +capsule+ builder yield symbols.
|
|
933
1062
|
# Entries are flat hashes with array/scalar values — no deep recursion.
|
|
934
1063
|
def normalize_entry(entry, group:)
|
|
935
|
-
raise
|
|
1064
|
+
raise Pgbus::ConfigurationError, "#{group} entry must be a Hash, got #{entry.class}: #{entry.inspect}" unless entry.is_a?(Hash)
|
|
936
1065
|
|
|
937
1066
|
entry.transform_keys(&:to_sym)
|
|
938
1067
|
end
|
|
@@ -977,13 +1106,13 @@ module Pgbus
|
|
|
977
1106
|
return if existing_queues.empty?
|
|
978
1107
|
|
|
979
1108
|
if existing_queues.include?(CapsuleDSL::WILDCARD)
|
|
980
|
-
raise
|
|
1109
|
+
raise Pgbus::ConfigurationError,
|
|
981
1110
|
"an existing named capsule already uses '*' (matches every queue) — " \
|
|
982
1111
|
"the new capsule's queues #{new_queues.inspect} would overlap with it"
|
|
983
1112
|
end
|
|
984
1113
|
|
|
985
1114
|
if new_queues.include?(CapsuleDSL::WILDCARD)
|
|
986
|
-
raise
|
|
1115
|
+
raise Pgbus::ConfigurationError,
|
|
987
1116
|
"the new capsule uses '*' (matches every queue) but other named capsules " \
|
|
988
1117
|
"are already defined with queues #{existing_queues.inspect} — " \
|
|
989
1118
|
"the wildcard would overlap with all of them"
|
|
@@ -992,7 +1121,7 @@ module Pgbus
|
|
|
992
1121
|
conflict = new_queues.find { |q| existing_queues.include?(q) }
|
|
993
1122
|
return unless conflict
|
|
994
1123
|
|
|
995
|
-
raise
|
|
1124
|
+
raise Pgbus::ConfigurationError,
|
|
996
1125
|
"queue #{conflict.inspect} is already assigned to another named capsule — " \
|
|
997
1126
|
"named capsules cannot share queues"
|
|
998
1127
|
end
|
|
@@ -1003,7 +1132,7 @@ module Pgbus
|
|
|
1003
1132
|
entries.sum do |entry|
|
|
1004
1133
|
threads = entry[:threads] || default_threads
|
|
1005
1134
|
unless threads.is_a?(Integer) && threads.positive?
|
|
1006
|
-
raise
|
|
1135
|
+
raise Pgbus::ConfigurationError,
|
|
1007
1136
|
"#{group} threads must be a positive integer, got #{threads.inspect}"
|
|
1008
1137
|
end
|
|
1009
1138
|
|
data/lib/pgbus/doctor.rb
CHANGED
|
@@ -5,7 +5,7 @@ require "pgbus/mcp/health_analyzer"
|
|
|
5
5
|
|
|
6
6
|
module Pgbus
|
|
7
7
|
# Preflight diagnostics for a pgbus deployment — the single command that
|
|
8
|
-
# answers "is this environment healthy enough to run?". Runs
|
|
8
|
+
# answers "is this environment healthy enough to run?". Runs seven checks and
|
|
9
9
|
# returns a machine-readable result plus a human report, so `pgbus doctor`
|
|
10
10
|
# and `rake pgbus:doctor` can gate a deploy or CI run (exit 0 on success,
|
|
11
11
|
# 1 on any failure).
|
|
@@ -40,7 +40,8 @@ module Pgbus
|
|
|
40
40
|
check_pgmq_schema,
|
|
41
41
|
check_queues,
|
|
42
42
|
check_notify,
|
|
43
|
-
check_processes
|
|
43
|
+
check_processes,
|
|
44
|
+
check_allowed_global_id_models
|
|
44
45
|
].map(&:to_h)
|
|
45
46
|
end
|
|
46
47
|
|
|
@@ -83,11 +84,17 @@ module Pgbus
|
|
|
83
84
|
|
|
84
85
|
private
|
|
85
86
|
|
|
87
|
+
# True in a Rails production environment. Guarded so the doctor still runs
|
|
88
|
+
# outside Rails (plain Ruby, tests) without assuming Rails is loaded.
|
|
89
|
+
def production?
|
|
90
|
+
defined?(Rails) && Rails.respond_to?(:env) && Rails.env.production?
|
|
91
|
+
end
|
|
92
|
+
|
|
86
93
|
# 1. Configuration validity.
|
|
87
94
|
def check_configuration
|
|
88
95
|
@config.validate!
|
|
89
96
|
Check.new(name: "Configuration", status: :ok, detail: "valid")
|
|
90
|
-
rescue ArgumentError => e
|
|
97
|
+
rescue Pgbus::ConfigurationError, ArgumentError => e
|
|
91
98
|
Check.new(name: "Configuration", status: :fail, detail: e.message)
|
|
92
99
|
rescue StandardError => e
|
|
93
100
|
Check.new(name: "Configuration", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
@@ -185,6 +192,24 @@ module Pgbus
|
|
|
185
192
|
Check.new(name: "Process liveness", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
186
193
|
end
|
|
187
194
|
|
|
195
|
+
# 7. GlobalID allowlist — security. allowed_global_id_models = nil means
|
|
196
|
+
# "allow ANY model as a GlobalID event/job argument", which lets a crafted
|
|
197
|
+
# payload deserialize arbitrary AR models. It's the default for upgrade
|
|
198
|
+
# continuity, so this is a warning (never a failure), and only in production
|
|
199
|
+
# where the blast radius is real.
|
|
200
|
+
def check_allowed_global_id_models
|
|
201
|
+
if @config.allowed_global_id_models.nil? && production?
|
|
202
|
+
return Check.new(name: "GlobalID allowlist", status: :warn,
|
|
203
|
+
detail: "allowed_global_id_models is nil (allow-all) in production — " \
|
|
204
|
+
"set an explicit allowlist of models permitted as GlobalID arguments")
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
Check.new(name: "GlobalID allowlist", status: :ok,
|
|
208
|
+
detail: @config.allowed_global_id_models.nil? ? "allow-all (non-production)" : "allowlist configured")
|
|
209
|
+
rescue StandardError => e
|
|
210
|
+
Check.new(name: "GlobalID allowlist", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
211
|
+
end
|
|
212
|
+
|
|
188
213
|
# Physical queue names known to PGMQ. list_queues rows may be Hashes (with a
|
|
189
214
|
# :queue_name key) or value objects responding to #queue_name.
|
|
190
215
|
def existing_queue_names
|
|
@@ -25,7 +25,7 @@ module Pgbus
|
|
|
25
25
|
|
|
26
26
|
def post(&block)
|
|
27
27
|
raise_if_fatal!
|
|
28
|
-
raise "Execution pool is shutting down" if shutdown?
|
|
28
|
+
raise Pgbus::ExecutionPoolError, "Execution pool is shutting down" if shutdown?
|
|
29
29
|
|
|
30
30
|
reserved = false
|
|
31
31
|
reserve_capacity!
|
|
@@ -193,7 +193,7 @@ module Pgbus
|
|
|
193
193
|
|
|
194
194
|
def reserve_capacity!
|
|
195
195
|
@mutex.synchronize do
|
|
196
|
-
raise "Execution pool is at capacity" if @available_capacity <= 0
|
|
196
|
+
raise Pgbus::ExecutionPoolError, "Execution pool is at capacity" if @available_capacity <= 0
|
|
197
197
|
|
|
198
198
|
@available_capacity -= 1
|
|
199
199
|
end
|