pgbus 0.6.8 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +166 -0
  3. data/lib/generators/pgbus/add_failed_events_index_generator.rb +4 -11
  4. data/lib/generators/pgbus/add_job_locks_generator.rb +4 -11
  5. data/lib/generators/pgbus/add_job_stats_generator.rb +4 -11
  6. data/lib/generators/pgbus/add_job_stats_latency_generator.rb +4 -11
  7. data/lib/generators/pgbus/add_job_stats_queue_index_generator.rb +4 -11
  8. data/lib/generators/pgbus/add_outbox_generator.rb +4 -11
  9. data/lib/generators/pgbus/add_presence_generator.rb +4 -11
  10. data/lib/generators/pgbus/add_queue_states_generator.rb +4 -11
  11. data/lib/generators/pgbus/add_recurring_generator.rb +4 -11
  12. data/lib/generators/pgbus/add_stream_stats_generator.rb +4 -11
  13. data/lib/generators/pgbus/install_generator.rb +4 -11
  14. data/lib/generators/pgbus/migrate_job_locks_generator.rb +4 -11
  15. data/lib/generators/pgbus/migration_path.rb +28 -0
  16. data/lib/generators/pgbus/tune_autovacuum_generator.rb +4 -11
  17. data/lib/generators/pgbus/upgrade_pgmq_generator.rb +4 -11
  18. data/lib/pgbus/active_job/executor.rb +3 -1
  19. data/lib/pgbus/circuit_breaker.rb +1 -1
  20. data/lib/pgbus/client.rb +10 -3
  21. data/lib/pgbus/configuration.rb +22 -0
  22. data/lib/pgbus/engine.rb +1 -0
  23. data/lib/pgbus/error_reporter.rb +48 -0
  24. data/lib/pgbus/failed_event_recorder.rb +1 -8
  25. data/lib/pgbus/log_formatter.rb +96 -0
  26. data/lib/pgbus/outbox/poller.rb +4 -4
  27. data/lib/pgbus/process/consumer.rb +5 -1
  28. data/lib/pgbus/process/dispatcher.rb +1 -1
  29. data/lib/pgbus/process/supervisor.rb +6 -6
  30. data/lib/pgbus/process/worker.rb +8 -3
  31. data/lib/pgbus/queue_name_validator.rb +28 -9
  32. data/lib/pgbus/streams/key.rb +173 -0
  33. data/lib/pgbus/streams/streamable.rb +57 -0
  34. data/lib/pgbus/streams.rb +37 -0
  35. data/lib/pgbus/version.rb +1 -1
  36. data/lib/pgbus.rb +14 -0
  37. data/lib/tasks/pgbus_autovacuum.rake +40 -0
  38. metadata +7 -1
@@ -2,11 +2,13 @@
2
2
 
3
3
  require "rails/generators"
4
4
  require "rails/generators/active_record"
5
+ require_relative "migration_path"
5
6
 
6
7
  module Pgbus
7
8
  module Generators
8
9
  class InstallGenerator < Rails::Generators::Base
9
10
  include ActiveRecord::Generators::Migration
11
+ include MigrationPath
10
12
 
11
13
  source_root File.expand_path("templates", __dir__)
12
14
 
@@ -25,13 +27,8 @@ module Pgbus
25
27
  "Migrations go to db/pgbus_migrate/ and schema to db/pgbus_schema.rb"
26
28
 
27
29
  def create_migration_file
28
- if separate_database?
29
- migration_template "migration.rb.erb",
30
- "db/pgbus_migrate/create_pgbus_tables.rb"
31
- else
32
- migration_template "migration.rb.erb",
33
- "db/migrate/create_pgbus_tables.rb"
34
- end
30
+ migration_template "migration.rb.erb",
31
+ File.join(pgbus_migrate_path, "create_pgbus_tables.rb")
35
32
  end
36
33
 
37
34
  def create_config_file
@@ -121,10 +118,6 @@ module Pgbus
121
118
  def database_name
122
119
  options[:database]
123
120
  end
124
-
125
- def separate_database?
126
- database_name.present?
127
- end
128
121
  end
129
122
  end
130
123
  end
@@ -2,11 +2,13 @@
2
2
 
3
3
  require "rails/generators"
4
4
  require "rails/generators/active_record"
5
+ require_relative "migration_path"
5
6
 
6
7
  module Pgbus
7
8
  module Generators
8
9
  class MigrateJobLocksGenerator < Rails::Generators::Base
9
10
  include ActiveRecord::Generators::Migration
11
+ include MigrationPath
10
12
 
11
13
  source_root File.expand_path("templates", __dir__)
12
14
 
@@ -18,13 +20,8 @@ module Pgbus
18
20
  desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
19
21
 
20
22
  def create_migration_file
21
- if separate_database?
22
- migration_template "migrate_job_locks_to_uniqueness_keys.rb.erb",
23
- "db/pgbus_migrate/migrate_pgbus_job_locks_to_uniqueness_keys.rb"
24
- else
25
- migration_template "migrate_job_locks_to_uniqueness_keys.rb.erb",
26
- "db/migrate/migrate_pgbus_job_locks_to_uniqueness_keys.rb"
27
- end
23
+ migration_template "migrate_job_locks_to_uniqueness_keys.rb.erb",
24
+ File.join(pgbus_migrate_path, "migrate_pgbus_job_locks_to_uniqueness_keys.rb")
28
25
  end
29
26
 
30
27
  def display_post_install
@@ -47,10 +44,6 @@ module Pgbus
47
44
  def migration_version
48
45
  "[#{ActiveRecord::Migration.current_version}]"
49
46
  end
50
-
51
- def separate_database?
52
- options[:database].present?
53
- end
54
47
  end
55
48
  end
56
49
  end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ module Generators
5
+ # Shared migration path logic for all pgbus generators.
6
+ #
7
+ # When --database is passed, uses Rails' built-in db_migrate_path which
8
+ # reads migrations_paths from database.yml. This is the standard Rails
9
+ # multi-database mechanism and respects custom paths like db/pgbus_migrate/.
10
+ #
11
+ # Falls back to db/migrate/ when no --database is set (single-database mode).
12
+ module MigrationPath
13
+ private
14
+
15
+ def pgbus_migrate_path
16
+ if separate_database?
17
+ db_migrate_path
18
+ else
19
+ "db/migrate"
20
+ end
21
+ end
22
+
23
+ def separate_database?
24
+ options[:database].present?
25
+ end
26
+ end
27
+ end
28
+ end
@@ -2,11 +2,13 @@
2
2
 
3
3
  require "rails/generators"
4
4
  require "rails/generators/active_record"
5
+ require_relative "migration_path"
5
6
 
6
7
  module Pgbus
7
8
  module Generators
8
9
  class TuneAutovacuumGenerator < Rails::Generators::Base
9
10
  include ActiveRecord::Generators::Migration
11
+ include MigrationPath
10
12
 
11
13
  source_root File.expand_path("templates", __dir__)
12
14
 
@@ -18,13 +20,8 @@ module Pgbus
18
20
  desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
19
21
 
20
22
  def create_migration_file
21
- if separate_database?
22
- migration_template "tune_autovacuum.rb.erb",
23
- "db/pgbus_migrate/tune_pgbus_autovacuum.rb"
24
- else
25
- migration_template "tune_autovacuum.rb.erb",
26
- "db/migrate/tune_pgbus_autovacuum.rb"
27
- end
23
+ migration_template "tune_autovacuum.rb.erb",
24
+ File.join(pgbus_migrate_path, "tune_pgbus_autovacuum.rb")
28
25
  end
29
26
 
30
27
  def display_post_install
@@ -46,10 +43,6 @@ module Pgbus
46
43
  def migration_version
47
44
  "[#{ActiveRecord::Migration.current_version}]"
48
45
  end
49
-
50
- def separate_database?
51
- options[:database].present?
52
- end
53
46
  end
54
47
  end
55
48
  end
@@ -2,11 +2,13 @@
2
2
 
3
3
  require "rails/generators"
4
4
  require "rails/generators/active_record"
5
+ require_relative "migration_path"
5
6
 
6
7
  module Pgbus
7
8
  module Generators
8
9
  class UpgradePgmqGenerator < Rails::Generators::Base
9
10
  include ActiveRecord::Generators::Migration
11
+ include MigrationPath
10
12
 
11
13
  source_root File.expand_path("templates", __dir__)
12
14
 
@@ -18,13 +20,8 @@ module Pgbus
18
20
  desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
19
21
 
20
22
  def create_migration_file
21
- if separate_database?
22
- migration_template "upgrade_pgmq.rb.erb",
23
- "db/pgbus_migrate/upgrade_pgmq_to_v#{target_version_slug}.rb"
24
- else
25
- migration_template "upgrade_pgmq.rb.erb",
26
- "db/migrate/upgrade_pgmq_to_v#{target_version_slug}.rb"
27
- end
23
+ migration_template "upgrade_pgmq.rb.erb",
24
+ File.join(pgbus_migrate_path, "upgrade_pgmq_to_v#{target_version_slug}.rb")
28
25
  end
29
26
 
30
27
  def display_post_upgrade
@@ -51,10 +48,6 @@ module Pgbus
51
48
  def target_version_slug
52
49
  target_version.tr(".", "_")
53
50
  end
54
-
55
- def separate_database?
56
- options[:database].present?
57
- end
58
51
  end
59
52
  end
60
53
  end
@@ -146,7 +146,9 @@ module Pgbus
146
146
  end
147
147
 
148
148
  def handle_failure(message, queue_name, error, payload: nil)
149
- Pgbus.logger.error { "[Pgbus] Job failed: #{error.class}: #{error.message}" }
149
+ ctx = { action: "execute_job", queue: queue_name, job_class: payload&.dig("job_class"),
150
+ msg_id: message.msg_id.to_i, read_ct: message.read_ct.to_i }
151
+ ErrorReporter.report(error, ctx)
150
152
  Pgbus.logger.debug { error.backtrace&.join("\n") }
151
153
 
152
154
  # Record failure for dashboard visibility.
@@ -92,7 +92,7 @@ module Pgbus
92
92
  @failure_counts.delete(queue_name)
93
93
  invalidate_cache(queue_name)
94
94
  rescue StandardError => e
95
- Pgbus.logger.error { "[Pgbus] Circuit breaker trip failed for #{queue_name}: #{e.message}" }
95
+ ErrorReporter.report(e, { action: "circuit_breaker_trip", queue: queue_name })
96
96
  end
97
97
 
98
98
  def check_paused(queue_name)
data/lib/pgbus/client.rb CHANGED
@@ -156,10 +156,17 @@ module Pgbus
156
156
  # Read from multiple queues in a single SQL query (UNION ALL).
157
157
  # Each returned message includes a queue_name field identifying its source.
158
158
  # queue_names should be logical names (prefix is added automatically).
159
- def read_multi(queue_names, qty:, vt: nil)
159
+ #
160
+ # `qty` is the per-queue cap (pgmq-ruby semantics), so without `limit:` the
161
+ # caller receives up to `queue_count * qty` messages. Pass `limit:` to cap
162
+ # the total across all queues — required when feeding a fixed-size pool,
163
+ # otherwise the pool can overflow on multi-queue reads (issue #123).
164
+ def read_multi(queue_names, qty:, vt: nil, limit: nil)
160
165
  full_names = queue_names.map { |q| config.queue_name(q) }
161
- Instrumentation.instrument("pgbus.client.read_multi", queues: full_names, qty: qty) do
162
- synchronized { @pgmq.read_multi(full_names, vt: vt || config.visibility_timeout, qty: qty) }
166
+ Instrumentation.instrument("pgbus.client.read_multi", queues: full_names, qty: qty, limit: limit) do
167
+ synchronized do
168
+ @pgmq.read_multi(full_names, vt: vt || config.visibility_timeout, qty: qty, limit: limit)
169
+ end
163
170
  end
164
171
  end
165
172
 
@@ -59,6 +59,11 @@ module Pgbus
59
59
 
60
60
  # Logging
61
61
  attr_accessor :logger
62
+ attr_reader :log_format # rubocop:disable Style/AccessorGrouping
63
+
64
+ # Error reporting — array of callable objects invoked on caught exceptions.
65
+ # Each receives (exception, context_hash) or (exception, context_hash, config).
66
+ attr_accessor :error_reporters
62
67
 
63
68
  # LISTEN/NOTIFY. Only the on/off switch is user-facing — the throttle
64
69
  # interval is a Postgres-side tuning knob that lives as a constant on
@@ -140,6 +145,8 @@ module Pgbus
140
145
  @allowed_global_id_models = nil # nil = allow all (for backwards compat)
141
146
 
142
147
  @logger = (defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger) || Logger.new($stdout)
148
+ @log_format = :text
149
+ @error_reporters = []
143
150
 
144
151
  @listen_notify = true
145
152
 
@@ -224,6 +231,21 @@ module Pgbus
224
231
  ExecutionPools.normalize_mode(mode)
225
232
  end
226
233
 
234
+ VALID_LOG_FORMATS = %i[text json].freeze
235
+
236
+ def log_format=(format)
237
+ format = format.to_sym
238
+ unless VALID_LOG_FORMATS.include?(format)
239
+ raise ArgumentError, "Invalid log_format: #{format}. Must be one of: #{VALID_LOG_FORMATS.join(", ")}"
240
+ end
241
+
242
+ @log_format = format
243
+ @logger.formatter = case format
244
+ when :json then LogFormatter::JSON.new
245
+ when :text then LogFormatter::Text.new
246
+ end
247
+ end
248
+
227
249
  VALID_PGMQ_SCHEMA_MODES = %i[auto extension embedded].freeze
228
250
 
229
251
  def pgmq_schema_mode=(mode)
data/lib/pgbus/engine.rb CHANGED
@@ -47,6 +47,7 @@ module Pgbus
47
47
  rake_tasks do
48
48
  load File.expand_path("../tasks/pgbus_pgmq.rake", __dir__)
49
49
  load File.expand_path("../tasks/pgbus_streams.rake", __dir__)
50
+ load File.expand_path("../tasks/pgbus_autovacuum.rake", __dir__)
50
51
  end
51
52
 
52
53
  initializer "pgbus.i18n" do
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ # Central error reporting module. Iterates all configured error reporters
5
+ # and logs the error. Inspired by Sidekiq's error_handlers pattern.
6
+ #
7
+ # Usage:
8
+ # Pgbus::ErrorReporter.report(exception, { queue: "default" })
9
+ #
10
+ # Configuration:
11
+ # Pgbus.configure do |c|
12
+ # c.error_reporters << ->(ex, ctx) { Appsignal.set_error(ex) { |t| t.set_tags(ctx) } }
13
+ # end
14
+ module ErrorReporter
15
+ module_function
16
+
17
+ def report(exception, context = {}, config: Pgbus.configuration)
18
+ log_error(exception, context, config: config)
19
+
20
+ config.error_reporters.each do |handler|
21
+ call_handler(handler, exception, context, config)
22
+ rescue Exception => e # rubocop:disable Lint/RescueException
23
+ config.logger.error { "[Pgbus] Error reporter raised: #{e.class}: #{e.message}" }
24
+ end
25
+ rescue Exception # rubocop:disable Lint/RescueException
26
+ # ErrorReporter must never raise — callers sit inside rescue blocks
27
+ # where an unexpected raise would break fault-tolerance invariants.
28
+ nil
29
+ end
30
+
31
+ def call_handler(handler, exception, context, config)
32
+ target = handler.is_a?(Proc) ? handler : handler.method(:call)
33
+ if target.arity == 3 || (target.arity.negative? && target.parameters.size >= 3)
34
+ handler.call(exception, context, config)
35
+ else
36
+ handler.call(exception, context)
37
+ end
38
+ end
39
+
40
+ def log_error(exception, context, config:)
41
+ config.logger.error do
42
+ msg = "[Pgbus] #{exception.class}: #{exception.message}"
43
+ msg += " (#{context.inspect})" unless context.empty?
44
+ msg
45
+ end
46
+ end
47
+ end
48
+ end
@@ -32,14 +32,7 @@ module Pgbus
32
32
  ]
33
33
  )
34
34
  rescue StandardError => e
35
- # ERROR-level: silent loss of failure-tracking data defeats the
36
- # purpose of the dashboard's "Failed Jobs" section. If recording
37
- # fails, surface it loudly so the broken state can be diagnosed
38
- # rather than silently masked.
39
- Pgbus.logger.error do
40
- "[Pgbus] Failed to record failed event for queue=#{queue_name} msg_id=#{msg_id}: " \
41
- "#{e.class}: #{e.message}"
42
- end
35
+ ErrorReporter.report(e, { action: "record_failed_event", queue: queue_name, msg_id: msg_id })
43
36
  end
44
37
 
45
38
  def clear!(queue_name:, msg_id:)
@@ -0,0 +1,96 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "logger"
5
+ require "time"
6
+
7
+ module Pgbus
8
+ # Log formatters for Pgbus, inspired by Sidekiq::Logger::Formatters.
9
+ #
10
+ # Usage:
11
+ # Pgbus.configure do |c|
12
+ # c.logger.formatter = Pgbus::LogFormatter::JSON.new
13
+ # end
14
+ #
15
+ # Or via the convenience config option:
16
+ # Pgbus.configure do |c|
17
+ # c.log_format = :json
18
+ # end
19
+ module LogFormatter
20
+ module_function
21
+
22
+ def tid
23
+ Thread.current[:pgbus_tid] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
24
+ end
25
+
26
+ # Thread-local context for structured logging. Works like
27
+ # Sidekiq::Context — any key/value pairs set via with_context
28
+ # appear in the JSON output under the "ctx" key.
29
+ def with_context(hash)
30
+ orig = current_context.dup
31
+ current_context.merge!(hash)
32
+ yield
33
+ ensure
34
+ Thread.current[:pgbus_log_context] = orig
35
+ end
36
+
37
+ def current_context
38
+ Thread.current[:pgbus_log_context] ||= {}
39
+ end
40
+
41
+ # Human-readable text formatter with Pgbus context.
42
+ # Output: "INFO 2024-01-15T10:30:00.000Z pid=1234 tid=abc queue=default: message\n"
43
+ class Text < ::Logger::Formatter
44
+ def call(severity, time, _progname, message)
45
+ "#{severity} #{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{LogFormatter.tid}#{format_context}: #{message}\n"
46
+ end
47
+
48
+ private
49
+
50
+ def format_context
51
+ ctx = LogFormatter.current_context
52
+ return "" if ctx.empty?
53
+
54
+ " #{ctx.map { |k, v| "#{k}=#{v}" }.join(" ")}"
55
+ end
56
+ end
57
+
58
+ # JSON formatter for structured logging. Each log line is a single
59
+ # JSON object followed by a newline. Extracts the [Pgbus::Component]
60
+ # prefix from messages into a separate "component" field.
61
+ #
62
+ # Output fields:
63
+ # ts — ISO 8601 timestamp with milliseconds
64
+ # pid — process ID
65
+ # tid — thread ID (short hex)
66
+ # lvl — severity (DEBUG/INFO/WARN/ERROR/FATAL)
67
+ # msg — the log message (with component prefix stripped)
68
+ # component — extracted from [Pgbus] or [Pgbus::Foo] prefix (optional)
69
+ # ctx — thread-local context hash (optional, only when non-empty)
70
+ class JSON < ::Logger::Formatter
71
+ COMPONENT_PREFIX = /\A\[([^\]]+)\]\s*/
72
+
73
+ def call(severity, time, _progname, message)
74
+ msg = message.to_s
75
+ hash = {
76
+ ts: time.utc.iso8601(3),
77
+ pid: ::Process.pid,
78
+ tid: LogFormatter.tid,
79
+ lvl: severity
80
+ }
81
+
82
+ if (match = msg.match(COMPONENT_PREFIX))
83
+ hash[:component] = match[1]
84
+ msg = msg.sub(COMPONENT_PREFIX, "")
85
+ end
86
+
87
+ hash[:msg] = msg
88
+
89
+ ctx = LogFormatter.current_context
90
+ hash[:ctx] = ctx unless ctx.empty?
91
+
92
+ "#{::JSON.generate(hash)}\n"
93
+ end
94
+ end
95
+ end
96
+ end
@@ -65,7 +65,7 @@ module Pgbus
65
65
  Pgbus.logger.debug { "[Pgbus] Outbox published #{published} entries" } if published.positive?
66
66
  published
67
67
  rescue StandardError => e
68
- Pgbus.logger.error { "[Pgbus] Outbox poll error: #{e.message}" }
68
+ ErrorReporter.report(e, { action: "outbox_poll" })
69
69
  0
70
70
  end
71
71
 
@@ -92,7 +92,7 @@ module Pgbus
92
92
  entry.update!(published_at: Time.current)
93
93
  true
94
94
  rescue StandardError => e
95
- Pgbus.logger.error { "[Pgbus] Failed to publish outbox entry #{entry.id}: #{e.message}" }
95
+ ErrorReporter.report(e, { action: "outbox_publish_topic", entry_id: entry.id })
96
96
  false
97
97
  end
98
98
 
@@ -112,7 +112,7 @@ module Pgbus
112
112
  group.each { |e| e.update!(published_at: now) }
113
113
  succeeded += group.size
114
114
  rescue StandardError => e
115
- Pgbus.logger.error { "[Pgbus] Failed to batch-publish #{group.size} outbox entries: #{e.message}" }
115
+ ErrorReporter.report(e, { action: "outbox_batch_publish", queue: queue, batch_size: group.size })
116
116
  # Fall back to individual publishing for this group
117
117
  group.each { |entry| succeeded += 1 if publish_single_queue(entry) }
118
118
  end
@@ -133,7 +133,7 @@ module Pgbus
133
133
  entry.update!(published_at: Time.current)
134
134
  true
135
135
  rescue StandardError => e
136
- Pgbus.logger.error { "[Pgbus] Failed to publish outbox entry #{entry.id}: #{e.message}" }
136
+ ErrorReporter.report(e, { action: "outbox_publish_queue", entry_id: entry.id })
137
137
  false
138
138
  end
139
139
 
@@ -98,8 +98,12 @@ module Pgbus
98
98
  # the next read will route to DLQ above.
99
99
  end
100
100
 
101
+ # `qty` is the total pool capacity. pgmq-ruby treats `qty:` as per-queue,
102
+ # so we also pass `limit: qty` to cap the total across all queues —
103
+ # otherwise we get `queue_count * qty` messages and overflow the
104
+ # execution pool, crashing the consumer fork (issue #123).
101
105
  def fetch_multi_consumer(qty)
102
- messages = Pgbus.client.read_multi(@queue_names, qty: qty) || []
106
+ messages = Pgbus.client.read_multi(@queue_names, qty: qty, limit: qty) || []
103
107
  prefix = "#{config.queue_prefix}_"
104
108
 
105
109
  messages.map do |m|
@@ -94,7 +94,7 @@ module Pgbus
94
94
  yield
95
95
  instance_variable_set(ivar, now)
96
96
  rescue StandardError => e
97
- Pgbus.logger.error { "[Pgbus] Dispatcher maintenance error: #{e.message}" }
97
+ ErrorReporter.report(e, { action: "dispatcher_maintenance", task: ivar.to_s.delete_prefix("@last_").delete_suffix("_at") })
98
98
  end
99
99
 
100
100
  def cleanup_processed_events
@@ -83,7 +83,7 @@ module Pgbus
83
83
  @forks[pid] = { type: :worker, config: worker_config }
84
84
  Pgbus.logger.info { "[Pgbus] Forked worker pid=#{pid} queues=#{queues.join(",")} mode=#{exec_mode}" }
85
85
  rescue Errno::EAGAIN, Errno::ENOMEM => e
86
- Pgbus.logger.error { "[Pgbus] Fork failed for worker: #{e.message}" }
86
+ ErrorReporter.report(e, { action: "fork_worker", queues: queues })
87
87
  end
88
88
 
89
89
  def fork_dispatcher
@@ -103,7 +103,7 @@ module Pgbus
103
103
  @forks[pid] = { type: :dispatcher }
104
104
  Pgbus.logger.info { "[Pgbus] Forked dispatcher pid=#{pid}" }
105
105
  rescue Errno::EAGAIN, Errno::ENOMEM => e
106
- Pgbus.logger.error { "[Pgbus] Fork failed for dispatcher: #{e.message}" }
106
+ ErrorReporter.report(e, { action: "fork_dispatcher" })
107
107
  end
108
108
 
109
109
  def boot_scheduler
@@ -132,7 +132,7 @@ module Pgbus
132
132
  @forks[pid] = { type: :scheduler }
133
133
  Pgbus.logger.info { "[Pgbus] Forked scheduler pid=#{pid}" }
134
134
  rescue Errno::EAGAIN, Errno::ENOMEM => e
135
- Pgbus.logger.error { "[Pgbus] Fork failed for scheduler: #{e.message}" }
135
+ ErrorReporter.report(e, { action: "fork_scheduler" })
136
136
  end
137
137
 
138
138
  def recurring_tasks_configured?
@@ -186,7 +186,7 @@ module Pgbus
186
186
  @forks[pid] = { type: :consumer, config: consumer_config }
187
187
  Pgbus.logger.info { "[Pgbus] Forked consumer pid=#{pid} topics=#{topics.join(",")}" }
188
188
  rescue Errno::EAGAIN, Errno::ENOMEM => e
189
- Pgbus.logger.error { "[Pgbus] Fork failed for consumer: #{e.message}" }
189
+ ErrorReporter.report(e, { action: "fork_consumer", topics: topics })
190
190
  end
191
191
 
192
192
  def boot_outbox_poller
@@ -212,7 +212,7 @@ module Pgbus
212
212
  @forks[pid] = { type: :outbox_poller }
213
213
  Pgbus.logger.info { "[Pgbus] Forked outbox poller pid=#{pid}" }
214
214
  rescue Errno::EAGAIN, Errno::ENOMEM => e
215
- Pgbus.logger.error { "[Pgbus] Fork failed for outbox poller: #{e.message}" }
215
+ ErrorReporter.report(e, { action: "fork_outbox_poller" })
216
216
  end
217
217
 
218
218
  def monitor_loop
@@ -282,7 +282,7 @@ module Pgbus
282
282
  def bootstrap_queues
283
283
  Pgbus.client.ensure_all_queues
284
284
  rescue StandardError => e
285
- Pgbus.logger.error { "[Pgbus] Failed to bootstrap queues: #{e.message}" }
285
+ ErrorReporter.report(e, { action: "bootstrap_queues" })
286
286
  end
287
287
 
288
288
  def load_rails_app
@@ -151,7 +151,7 @@ module Pgbus
151
151
  if undefined_queue_table_error?(e)
152
152
  evict_missing_queues(e)
153
153
  else
154
- Pgbus.logger.error { "[Pgbus] Error fetching messages: #{e.message}" }
154
+ ErrorReporter.report(e, { action: "fetch_messages", queues: active_queues })
155
155
  end
156
156
  []
157
157
  end
@@ -194,8 +194,13 @@ module Pgbus
194
194
  # Use pgmq-ruby's read_multi to read from all queues in a single
195
195
  # SQL query (UNION ALL). Each returned message carries a queue_name
196
196
  # field so we can map it back to the logical queue.
197
+ #
198
+ # `qty` is the total pool capacity. pgmq-ruby treats `qty:` as per-queue,
199
+ # so we also pass `limit: qty` to cap the total across all queues —
200
+ # otherwise we get `queue_count * qty` messages and overflow the
201
+ # execution pool, crashing the worker fork (issue #123).
197
202
  def fetch_multi(active_queues, qty)
198
- messages = Pgbus.client.read_multi(active_queues, qty: qty) || []
203
+ messages = Pgbus.client.read_multi(active_queues, qty: qty, limit: qty) || []
199
204
  prefix = "#{config.queue_prefix}_"
200
205
 
201
206
  messages.map do |m|
@@ -223,7 +228,7 @@ module Pgbus
223
228
  @jobs_failed.increment
224
229
  @rate_counter.increment(:failed)
225
230
  @circuit_breaker.record_failure(queue_name)
226
- Pgbus.logger.error { "[Pgbus] Unhandled error processing message: #{e.message}" }
231
+ ErrorReporter.report(e, { action: "process_message", queue: queue_name })
227
232
  ensure
228
233
  @in_flight.decrement
229
234
  end
@@ -7,9 +7,21 @@ module Pgbus
7
7
  # (e.g., pgmq.q_<name>, pgmq.a_<name>). This module enforces strict
8
8
  # validation to prevent SQL injection via crafted queue names.
9
9
  module QueueNameValidator
10
- # PostgreSQL identifier limit is 63 bytes (NAMEDATALEN - 1).
11
- # PGMQ prefixes with "q_" or "a_" (2 chars), so limit the name itself.
12
- MAX_QUEUE_NAME_LENGTH = 61
10
+ # PostgreSQL's NAMEDATALEN caps identifiers at 63 bytes, but the
11
+ # effective limit is tighter: pgmq-ruby (our transport gem) rejects
12
+ # any queue name with `length >= 48` in `PGMQ::Client#validate_queue_name!`.
13
+ # That leaves an actual usable ceiling of 47 characters for the
14
+ # fully-prefixed name (`<queue_prefix>_<logical_name>`), which is what
15
+ # this constant expresses. pgmq-ruby picked 48 to leave headroom for
16
+ # PGMQ's internal tables (`pgmq.q_`, `pgmq.a_`, sequences, indexes)
17
+ # which all get suffixed beyond the base name.
18
+ #
19
+ # Historically this was 61 (the raw PostgreSQL ceiling minus PGMQ's
20
+ # `q_`/`a_` prefix). That was wrong in practice: names in the 48-61
21
+ # range passed pgbus's validator but blew up deep inside pgmq-ruby
22
+ # with an InvalidQueueNameError — exactly the kind of opaque failure
23
+ # the validator was meant to catch up front.
24
+ MAX_QUEUE_NAME_LENGTH = 47
13
25
 
14
26
  # Only alphanumeric characters and underscores are allowed.
15
27
  VALID_QUEUE_NAME_PATTERN = /\A[a-zA-Z0-9_]+\z/
@@ -34,16 +46,23 @@ module Pgbus
34
46
  name
35
47
  end
36
48
 
37
- # Normalizes a queue name by replacing common separators (hyphens, dots)
38
- # with underscores, stripping remaining invalid characters, and collapsing
39
- # consecutive underscores. Use this for names from external sources
40
- # (e.g., Turbo stream names like "hotwire-livereload") where the intent
41
- # is to derive a valid PGMQ queue name that preserves readability.
49
+ # Normalizes a queue name by replacing common separators (hyphens,
50
+ # dots, colons) with underscores, stripping remaining invalid
51
+ # characters, and collapsing consecutive underscores. Use this for
52
+ # names from external sources (e.g., Turbo stream names like
53
+ # "hotwire-livereload" or "gid://app/Foo/1") where the intent is
54
+ # to derive a valid PGMQ queue name that preserves as much of the
55
+ # original identifier as possible.
56
+ #
57
+ # Colons in particular are the turbo-rails stream-name separator
58
+ # (`Pgbus.stream([user, :notifications])` → `"user_gid:notifications"`),
59
+ # so they must map to a safe character rather than be stripped —
60
+ # otherwise `"a:b"` and `"ab"` would collide on the same queue.
42
61
  def normalize(name)
43
62
  name = name.to_s
44
63
  return validate!(name) if VALID_QUEUE_NAME_PATTERN.match?(name)
45
64
 
46
- normalized = name.gsub(/[-.]/, "_") # hyphens/dots → underscores
65
+ normalized = name.gsub(/[-.:]/, "_") # hyphens/dots/colons → underscores
47
66
  .gsub(/[^a-zA-Z0-9_]/, "") # strip remaining invalid chars
48
67
  .gsub(/_+/, "_") # collapse consecutive underscores
49
68
  .gsub(/\A_|_\z/, "") # strip leading/trailing underscores