pgbus 0.17.0 → 0.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +50 -0
  3. data/README.md +69 -14
  4. data/Rakefile +7 -1
  5. data/app/controllers/concerns/pgbus/job_listing.rb +37 -0
  6. data/app/controllers/concerns/pgbus/process_listing.rb +17 -0
  7. data/app/controllers/pgbus/application_controller.rb +5 -2
  8. data/app/controllers/pgbus/batches_controller.rb +4 -1
  9. data/app/controllers/pgbus/dashboard_controller.rb +20 -6
  10. data/app/controllers/pgbus/dead_letter_controller.rb +27 -3
  11. data/app/controllers/pgbus/events_controller.rb +52 -3
  12. data/app/controllers/pgbus/insights_controller.rb +19 -1
  13. data/app/controllers/pgbus/jobs_controller.rb +45 -26
  14. data/app/controllers/pgbus/locks_controller.rb +7 -2
  15. data/app/controllers/pgbus/outbox_controller.rb +4 -1
  16. data/app/controllers/pgbus/processes_controller.rb +3 -1
  17. data/app/controllers/pgbus/queues_controller.rb +16 -4
  18. data/app/controllers/pgbus/recurring_tasks_controller.rb +5 -7
  19. data/app/frontend/pgbus/modules/charts.js +119 -77
  20. data/app/frontend/pgbus/style.css +1 -1
  21. data/app/frontend/pgbus/tailwind.css +18 -0
  22. data/app/helpers/pgbus/application_helper.rb +155 -113
  23. data/app/helpers/pgbus/button_helper.rb +88 -0
  24. data/app/helpers/pgbus/dashboard_helper.rb +157 -0
  25. data/app/helpers/pgbus/events_helper.rb +40 -0
  26. data/app/helpers/pgbus/insights_helper.rb +39 -0
  27. data/app/helpers/pgbus/processes_helper.rb +79 -0
  28. data/app/views/pgbus/batches/_batches_table.html.erb +16 -13
  29. data/app/views/pgbus/batches/show.html.erb +3 -4
  30. data/app/views/pgbus/dashboard/_processes_table.html.erb +11 -11
  31. data/app/views/pgbus/dashboard/_queue_health.html.erb +42 -51
  32. data/app/views/pgbus/dashboard/_queues_table.html.erb +18 -16
  33. data/app/views/pgbus/dashboard/_recent_failures.html.erb +16 -12
  34. data/app/views/pgbus/dashboard/_stats_cards.html.erb +70 -36
  35. data/app/views/pgbus/dashboard/show.html.erb +1 -1
  36. data/app/views/pgbus/dead_letter/_error.html.erb +13 -0
  37. data/app/views/pgbus/dead_letter/_messages_table.html.erb +37 -96
  38. data/app/views/pgbus/dead_letter/_row.html.erb +75 -0
  39. data/app/views/pgbus/dead_letter/index.html.erb +6 -9
  40. data/app/views/pgbus/dead_letter/show.html.erb +37 -21
  41. data/app/views/pgbus/events/_list.html.erb +55 -0
  42. data/app/views/pgbus/events/_processed_table.html.erb +45 -0
  43. data/app/views/pgbus/events/_replay.html.erb +10 -0
  44. data/app/views/pgbus/events/_row.html.erb +162 -0
  45. data/app/views/pgbus/events/_tabs.html.erb +13 -0
  46. data/app/views/pgbus/events/index.html.erb +27 -48
  47. data/app/views/pgbus/events/show.html.erb +25 -7
  48. data/app/views/pgbus/insights/_chart_card.html.erb +21 -0
  49. data/app/views/pgbus/insights/_latency_table.html.erb +20 -0
  50. data/app/views/pgbus/insights/_status_table.html.erb +19 -0
  51. data/app/views/pgbus/insights/_throughput_table.html.erb +18 -0
  52. data/app/views/pgbus/insights/show.html.erb +83 -100
  53. data/app/views/pgbus/jobs/_list.html.erb +64 -0
  54. data/app/views/pgbus/jobs/_row.html.erb +107 -0
  55. data/app/views/pgbus/jobs/_tabs.html.erb +14 -0
  56. data/app/views/pgbus/jobs/index.html.erb +10 -10
  57. data/app/views/pgbus/jobs/show.html.erb +18 -20
  58. data/app/views/pgbus/locks/_concurrency.html.erb +25 -23
  59. data/app/views/pgbus/locks/_uniqueness.html.erb +20 -22
  60. data/app/views/pgbus/outbox/index.html.erb +30 -27
  61. data/app/views/pgbus/processes/_processes_table.html.erb +6 -25
  62. data/app/views/pgbus/processes/_row.html.erb +48 -0
  63. data/app/views/pgbus/processes/_status.html.erb +4 -0
  64. data/app/views/pgbus/queues/_queues_list.html.erb +28 -32
  65. data/app/views/pgbus/queues/show.html.erb +69 -118
  66. data/app/views/pgbus/recurring_tasks/_tasks_table.html.erb +75 -76
  67. data/app/views/pgbus/recurring_tasks/index.html.erb +1 -1
  68. data/app/views/pgbus/recurring_tasks/show.html.erb +45 -51
  69. data/app/views/pgbus/shared/_empty_row.html.erb +2 -0
  70. data/app/views/pgbus/shared/_pager.html.erb +34 -0
  71. data/config/i18n-tasks.yml +11 -2
  72. data/config/locales/da.yml +405 -113
  73. data/config/locales/de.yml +405 -113
  74. data/config/locales/en.yml +405 -113
  75. data/config/locales/es.yml +404 -112
  76. data/config/locales/fi.yml +404 -112
  77. data/config/locales/fr.yml +406 -114
  78. data/config/locales/it.yml +405 -113
  79. data/config/locales/ja.yml +405 -113
  80. data/config/locales/nb.yml +405 -113
  81. data/config/locales/nl.yml +405 -113
  82. data/config/locales/pt.yml +406 -114
  83. data/config/locales/sv.yml +404 -112
  84. data/lib/generators/pgbus/templates/initializer.rb.erb +4 -0
  85. data/lib/pgbus/active_job/executor.rb +36 -17
  86. data/lib/pgbus/cli/dlq.rb +32 -5
  87. data/lib/pgbus/client.rb +7 -2
  88. data/lib/pgbus/configuration.rb +93 -2
  89. data/lib/pgbus/dead_letter_header.rb +129 -0
  90. data/lib/pgbus/doctor.rb +37 -5
  91. data/lib/pgbus/event_bus/handler.rb +18 -11
  92. data/lib/pgbus/event_bus/registry.rb +6 -0
  93. data/lib/pgbus/failed_event_recorder.rb +51 -0
  94. data/lib/pgbus/instrumentation.rb +2 -1
  95. data/lib/pgbus/mcp/base_tool.rb +25 -0
  96. data/lib/pgbus/mcp/tools/concurrency_tool.rb +23 -4
  97. data/lib/pgbus/mcp/tools/dlq_detail_tool.rb +4 -2
  98. data/lib/pgbus/mcp/tools/dlq_tool.rb +10 -4
  99. data/lib/pgbus/mcp/tools/locks_tool.rb +23 -5
  100. data/lib/pgbus/mcp/tools/recurring_tool.rb +33 -4
  101. data/lib/pgbus/outbox/poller.rb +1 -1
  102. data/lib/pgbus/process/claim_buffer.rb +115 -0
  103. data/lib/pgbus/process/consumer.rb +94 -17
  104. data/lib/pgbus/process/dispatcher.rb +1 -1
  105. data/lib/pgbus/process/supervisor.rb +35 -12
  106. data/lib/pgbus/process/worker.rb +91 -25
  107. data/lib/pgbus/recurring/scheduler.rb +2 -1
  108. data/lib/pgbus/ruby_jit.rb +21 -0
  109. data/lib/pgbus/version.rb +1 -1
  110. data/lib/pgbus/visibility_heartbeat.rb +38 -3
  111. data/lib/pgbus/web/data_source/dashboard_signals.rb +170 -0
  112. data/lib/pgbus/web/data_source/dead_letter.rb +142 -0
  113. data/lib/pgbus/web/data_source/event_list.rb +161 -0
  114. data/lib/pgbus/web/data_source/job_list.rb +323 -0
  115. data/lib/pgbus/web/data_source/list_counts.rb +80 -0
  116. data/lib/pgbus/web/data_source/process_list.rb +69 -0
  117. data/lib/pgbus/web/data_source/queue_summary.rb +122 -0
  118. data/lib/pgbus/web/data_source.rb +132 -254
  119. data/lib/pgbus/web/dead_letter_reason.rb +81 -0
  120. data/lib/pgbus/web/event_state.rb +71 -0
  121. data/lib/pgbus/web/health_app.rb +7 -3
  122. data/lib/pgbus/web/health_verdict.rb +78 -0
  123. data/lib/pgbus/web/insights_charts.rb +84 -0
  124. data/lib/pgbus/web/job_state.rb +111 -0
  125. data/lib/pgbus/web/process_activity.rb +97 -0
  126. data/lib/pgbus/web/process_state.rb +187 -0
  127. data/lib/pgbus/web/queue_summary.rb +104 -0
  128. data/lib/pgbus/web/time_format.rb +128 -0
  129. metadata +45 -4
  130. data/app/views/pgbus/events/_pending_table.html.erb +0 -169
  131. data/app/views/pgbus/jobs/_enqueued_table.html.erb +0 -118
  132. data/app/views/pgbus/jobs/_failed_table.html.erb +0 -76
@@ -36,6 +36,10 @@ Pgbus.configure do |c|
36
36
  c.workers = []
37
37
  c.capsule :default, queues: %w[critical default], threads: 5
38
38
  c.capsule :low, queues: %w[low], threads: 2
39
+ # CPU-bound jobs (rendering, reports, PDFs): threads share one GVL, so use
40
+ # processes instead. Capsule-level recycle limits override the global ones.
41
+ # c.capsule :render, queues: %w[render], threads: 1, processes: 4,
42
+ # max_memory_mb: 1_536
39
43
 
40
44
  # --- Worker recycling (prevents memory bloat) --------------------------
41
45
  c.max_jobs_per_worker = 10_000
@@ -19,15 +19,17 @@ module Pgbus
19
19
 
20
20
  def execute(message, queue_name, source_queue: nil)
21
21
  execution_start = monotonic_now
22
- tag = "msg_id=#{message.msg_id} queue=#{queue_name} read_ct=#{message.read_ct}"
23
- Pgbus.logger.debug { "[Pgbus::Executor] start #{tag}" }
22
+ Pgbus.logger.debug { "[Pgbus::Executor] start #{log_tag(message, queue_name)}" }
24
23
 
25
24
  payload = JSON.parse(message.message)
26
25
  job_class = payload["job_class"]
27
26
  read_count = message.read_ct.to_i
28
27
 
29
28
  if read_count > config.max_retries
30
- handle_dead_letter(message, queue_name, payload, source_queue: source_queue)
29
+ # Read the last error before clear! deletes the only row that has
30
+ # it: the DLQ copy carries it in its headers (issue #495).
31
+ last_error = FailedEventRecorder.last_error(queue_name: queue_name, msg_id: message.msg_id.to_i)
32
+ handle_dead_letter(message, queue_name, payload, source_queue: source_queue, last_error: last_error)
31
33
  FailedEventRecorder.clear!(queue_name: queue_name, msg_id: message.msg_id.to_i)
32
34
  signal_concurrency(payload)
33
35
  signal_batch_discarded(payload)
@@ -40,9 +42,10 @@ module Pgbus
40
42
  job_id: payload["job_id"],
41
43
  provider_job_id: payload["provider_job_id"],
42
44
  read_ct: read_count,
43
- msg_id: message.msg_id.to_i
45
+ msg_id: message.msg_id.to_i,
46
+ error: last_error&.dig(:error_class)
44
47
  )
45
- Pgbus.logger.debug { "[Pgbus::Executor] dead_lettered #{tag} job_class=#{job_class}" }
48
+ Pgbus.logger.debug { "[Pgbus::Executor] dead_lettered #{log_tag(message, queue_name)} job_class=#{job_class}" }
46
49
  return :dead_lettered
47
50
  end
48
51
  uniqueness_key = Uniqueness.extract_key(payload)
@@ -70,7 +73,7 @@ module Pgbus
70
73
  end
71
74
  end
72
75
 
73
- Pgbus.logger.debug { "[Pgbus::Executor] deserialized #{tag} job_class=#{job_class}" }
76
+ Pgbus.logger.debug { "[Pgbus::Executor] deserialized #{log_tag(message, queue_name)} job_class=#{job_class}" }
74
77
  job_succeeded = false
75
78
  retried = false
76
79
 
@@ -95,13 +98,13 @@ module Pgbus
95
98
  # job data, so hand it to the job before perform — that is what makes
96
99
  # `batch` (and `batch.enqueue` for open batches) work inside a job.
97
100
  assign_batch_id(job, payload)
98
- Pgbus.logger.debug { "[Pgbus::Executor] running #{tag} job_class=#{job_class}" }
101
+ Pgbus.logger.debug { "[Pgbus::Executor] running #{log_tag(message, queue_name)} job_class=#{job_class}" }
99
102
  with_visibility_heartbeat(job, queue_name, msg_id, source_queue, payload) { execute_job(job) }
100
103
  # retry_on re-enqueues from inside perform_now and returns normally:
101
104
  # this attempt is done (archive it) but the job is not — the retry
102
105
  # message carries the batch tag and signals on its own outcome.
103
106
  retried = Batch.retry_reenqueued?(payload["job_id"])
104
- Pgbus.logger.debug { "[Pgbus::Executor] perform_returned #{tag} job_class=#{job_class}" }
107
+ Pgbus.logger.debug { "[Pgbus::Executor] perform_returned #{log_tag(message, queue_name)} job_class=#{job_class}" }
105
108
  # Archiving is the exact-once claim on this execution.
106
109
  # :already_archived means another worker archived the message (our
107
110
  # heartbeat lapsed and it was redelivered): that worker owns the
@@ -109,20 +112,24 @@ module Pgbus
109
112
  # concurrency slot a second time (rails/solid_queue#761).
110
113
  if archive_from(queue_name, msg_id, source_queue: source_queue) == :already_archived
111
114
  Pgbus.logger.warn do
112
- "[Pgbus::Executor] already archived elsewhere, skipping signals #{tag} job_class=#{job_class}"
115
+ "[Pgbus::Executor] already archived elsewhere, skipping signals #{log_tag(message, queue_name)} job_class=#{job_class}"
113
116
  end
114
117
  release_duplicate_execution_lock(uniqueness_key, uniqueness_strategy, queue_name, msg_id)
115
118
  return :duplicate
116
119
  end
117
- Pgbus.logger.debug { "[Pgbus::Executor] archived #{tag} job_class=#{job_class}" }
120
+ Pgbus.logger.debug { "[Pgbus::Executor] archived #{log_tag(message, queue_name)} job_class=#{job_class}" }
118
121
  job_succeeded = true
119
122
  release_uniqueness_lock(uniqueness_key)
120
- FailedEventRecorder.clear!(queue_name: queue_name, msg_id: msg_id)
123
+ # Only FailedEventRecorder.record! writes this table, and only after a
124
+ # failed attempt, so a first delivery has no row: skip the per-job
125
+ # DELETE round trip (issue #484). A stale row from a dropped-and-
126
+ # recreated queue reusing this msg_id is the dispatcher's to sweep.
127
+ FailedEventRecorder.clear!(queue_name: queue_name, msg_id: msg_id) if read_count > 1
121
128
  end
122
129
 
123
130
  instrument("pgbus.job_completed", queue: queue_name, job_class: job_class)
124
131
  record_stat(payload, queue_name, "success", execution_start, message: message)
125
- Pgbus.logger.debug { "[Pgbus::Executor] done #{tag} job_class=#{job_class}" }
132
+ Pgbus.logger.debug { "[Pgbus::Executor] done #{log_tag(message, queue_name)} job_class=#{job_class}" }
126
133
  :success
127
134
  rescue *FATAL_EXCEPTIONS
128
135
  # Process-fatal: propagate so the supervisor/OS can react.
@@ -150,7 +157,7 @@ module Pgbus
150
157
  exception_object: e
151
158
  )
152
159
  record_stat(payload, queue_name, "failed", execution_start, message: message)
153
- Pgbus.logger.debug { "[Pgbus::Executor] failed #{tag} job_class=#{payload&.dig("job_class")} error=#{e.class}" }
160
+ Pgbus.logger.debug { "[Pgbus::Executor] failed #{log_tag(message, queue_name)} job_class=#{payload&.dig("job_class")} error=#{e.class}" }
154
161
  # Don't signal concurrency on transient failure — the job will be retried.
155
162
  # Semaphore is released only on success or dead-lettering.
156
163
  :failed
@@ -170,6 +177,12 @@ module Pgbus
170
177
 
171
178
  private
172
179
 
180
+ # Built inside the logger blocks so a job pays for the string only when
181
+ # debug logging is on (issue #484).
182
+ def log_tag(message, queue_name)
183
+ "msg_id=#{message.msg_id} queue=#{queue_name} read_ct=#{message.read_ct}"
184
+ end
185
+
173
186
  def assign_batch_id(job, payload)
174
187
  batch_id = payload[Batch::METADATA_KEY]
175
188
  return unless batch_id && job.respond_to?(:batch_id=)
@@ -419,20 +432,26 @@ module Pgbus
419
432
  defined?(PGMQ::Errors::ConnectionError) && error.is_a?(PGMQ::Errors::ConnectionError)
420
433
  end
421
434
 
422
- def handle_dead_letter(message, queue_name, payload, source_queue: nil)
435
+ def handle_dead_letter(message, queue_name, payload, source_queue: nil, last_error: nil)
423
436
  Pgbus.logger.warn do
424
437
  job_class = payload["job_class"] || "unknown"
425
- "[Pgbus] Moving job #{job_class} to dead letter queue after #{message.read_ct} attempts"
438
+ error = last_error ? " (last error: #{last_error[:error_class]})" : ""
439
+ "[Pgbus] Moving job #{job_class} to dead letter queue after #{message.read_ct} attempts#{error}"
426
440
  end
441
+ headers = DeadLetterHeader.build(
442
+ existing: message.headers, reason: DeadLetterHeader::REASON_MAX_RETRIES, source: "worker",
443
+ source_queue: source_queue || config.queue_name(queue_name), attempts: message.read_ct.to_i,
444
+ max_retries: config.max_retries, error: last_error
445
+ )
427
446
  if source_queue
428
447
  client.ensure_dead_letter_queue(queue_name)
429
448
  dlq_name = config.dead_letter_queue_name(queue_name)
430
449
  client.transaction do |txn|
431
- txn.produce(dlq_name, message.message, headers: message.headers)
450
+ txn.produce(dlq_name, message.message, headers: headers)
432
451
  txn.delete(source_queue, message.msg_id.to_i)
433
452
  end
434
453
  else
435
- client.move_to_dead_letter(queue_name, message)
454
+ client.move_to_dead_letter(queue_name, message, headers: headers)
436
455
  end
437
456
  end
438
457
  end
data/lib/pgbus/cli/dlq.rb CHANGED
@@ -12,7 +12,8 @@ module Pgbus
12
12
  module DLQ
13
13
  module_function
14
14
 
15
- ROW_FORMAT = "%-10s %-32s %-28s %-8s %s"
15
+ ROW_FORMAT = "%-10s %-32s %-28s %-8s %-26s %s"
16
+ REASON_WIDTH = 40
16
17
 
17
18
  def start(args, data_source: Pgbus::Web::DataSource.new)
18
19
  subcommand = args.first
@@ -41,7 +42,7 @@ module Pgbus
41
42
  end
42
43
 
43
44
  print_list_table(messages)
44
- puts "-" * 96
45
+ puts "-" * 140
45
46
  puts "Total dead-letter messages: #{data_source.dlq_total_count} " \
46
47
  "(page #{options[:page]}, #{options[:per_page]} per page)"
47
48
  end
@@ -56,6 +57,7 @@ module Pgbus
56
57
  puts "origin: #{origin_queue(detail[:queue_name])}"
57
58
  puts "read_ct: #{detail[:read_ct]}"
58
59
  puts "enqueued_at: #{detail[:enqueued_at]}"
60
+ print_dead_letter(Pgbus::Web::DeadLetterReason.present(detail))
59
61
  puts "payload:"
60
62
  puts Pgbus::Web::PayloadFilter.filter_json(detail[:message])
61
63
  end
@@ -120,14 +122,39 @@ module Pgbus
120
122
  end
121
123
 
122
124
  def print_list_table(messages)
123
- puts format(ROW_FORMAT, "MSG_ID", "DLQ QUEUE", "ORIGIN QUEUE", "READ_CT", "ENQUEUED_AT")
124
- puts "-" * 96
125
+ puts format(ROW_FORMAT, "MSG_ID", "DLQ QUEUE", "ORIGIN QUEUE", "READ_CT", "ENQUEUED_AT", "REASON")
126
+ puts "-" * 140
125
127
  messages.each do |m|
128
+ reason = Pgbus::Web::DeadLetterReason.present(m)
126
129
  puts format(ROW_FORMAT, m[:msg_id], m[:queue_name], origin_queue(m[:queue_name]),
127
- m[:read_ct], m[:enqueued_at])
130
+ m[:read_ct], m[:enqueued_at], (reason.error_class || reason.reason_key).to_s[0, REASON_WIDTH])
128
131
  end
129
132
  end
130
133
 
134
+ # Why the message died, from its pgbus_dead_letter header (issue #495).
135
+ def print_dead_letter(result)
136
+ if result.legacy?
137
+ puts "reason: not recorded (dead-lettered before pgbus #{Pgbus::DeadLetterHeader::SINCE})"
138
+ return
139
+ end
140
+
141
+ puts "died: #{result.died_at}"
142
+ puts "attempts: #{result.attempts}/#{result.max_retries}"
143
+ puts "source: #{result.source_queue}"
144
+ puts "reason: #{result.reason}"
145
+ puts "error: #{dead_letter_error(result)}"
146
+ puts "retried: #{result.retried_before} time(s) from the DLQ before" if result.retried_before
147
+ end
148
+
149
+ def dead_letter_error(result)
150
+ return "none recorded" unless result.error_class
151
+
152
+ error = "#{result.error_class}: #{result.error_message} (attempt #{result.error_attempt})"
153
+ # A stored message is untrusted: newlines would break the layout and
154
+ # ANSI sequences could rewrite the operator's terminal.
155
+ error.gsub(/[\x00-\x1f\x7f]/) { |char| format("\\x%02X", char.ord) }
156
+ end
157
+
131
158
  def origin_queue(dlq_queue_name)
132
159
  dlq_queue_name.delete_suffix(Pgbus::DEAD_LETTER_SUFFIX)
133
160
  end
data/lib/pgbus/client.rb CHANGED
@@ -592,7 +592,9 @@ module Pgbus
592
592
  end
593
593
  end
594
594
 
595
- def move_to_dead_letter(queue_name, message)
595
+ # headers: what the DLQ copy carries. Movers pass the message's headers
596
+ # merged with a DeadLetterHeader block; the default copies them verbatim.
597
+ def move_to_dead_letter(queue_name, message, headers: message.headers)
596
598
  dlq_name = config.dead_letter_queue_name(queue_name)
597
599
  full_queue = config.queue_name(queue_name)
598
600
 
@@ -600,7 +602,7 @@ module Pgbus
600
602
  ensure_dead_letter_queue(queue_name)
601
603
  synchronized do
602
604
  @pgmq.transaction do |txn|
603
- txn.produce(dlq_name, message.message, headers: message.headers)
605
+ txn.produce(dlq_name, message.message, headers: headers)
604
606
  txn.delete(full_queue, message.msg_id.to_i)
605
607
  end
606
608
  end
@@ -699,6 +701,9 @@ module Pgbus
699
701
  synchronized { @pgmq.drop_queue(name) }
700
702
  end
701
703
  @queues_created.delete(name)
704
+ # Failed-event rows are keyed by the name the executor saw (physical or
705
+ # logical); a recreated queue reuses msg_ids, so neither may survive.
706
+ FailedEventRecorder.clear_queue!([name, name.delete_prefix("#{config.queue_prefix}_")].uniq)
702
707
  result
703
708
  end
704
709
 
@@ -12,6 +12,12 @@ module Pgbus
12
12
 
13
13
  # Worker settings
14
14
  attr_accessor :polling_interval, :prefetch_limit, :execution_mode
15
+ # read_ahead (issue #486): how many messages a worker or event consumer
16
+ # claims beyond its free threads and holds, heartbeated, until a thread
17
+ # frees. Under database latency one read then feeds many jobs instead of
18
+ # one. 0 (default) is off. Overridable per capsule / event_consumers entry
19
+ # (`read_ahead:`); prefetch_limit still caps everything claimed.
20
+ attr_reader :read_ahead
15
21
  # visibility_heartbeat / visibility_heartbeat_interval: while a job runs,
16
22
  # its message's visibility timeout is re-armed every interval seconds
17
23
  # (default: a third of visibility_timeout), so a job that outlives the
@@ -277,6 +283,7 @@ module Pgbus
277
283
  @visibility_heartbeat_interval = nil
278
284
 
279
285
  @prefetch_limit = nil
286
+ @read_ahead = 0
280
287
  @execution_mode = :threads
281
288
 
282
289
  @max_jobs_per_worker = nil
@@ -573,6 +580,31 @@ module Pgbus
573
580
  (0...priority_levels).map { |p| priority_queue_name(name, p) }
574
581
  end
575
582
 
583
+ def read_ahead=(value)
584
+ @read_ahead = value.nil? ? 0 : value
585
+ end
586
+
587
+ # Read-ahead for one capsule or event_consumers entry, falling back to
588
+ # the global read_ahead (mirrors execution_mode_for).
589
+ def read_ahead_for(entry)
590
+ entry.fetch(:read_ahead, nil) || read_ahead
591
+ end
592
+
593
+ # Recycle limits a capsule may set for its own workers (issue #503).
594
+ RECYCLE_LIMIT_KEYS = %i[max_jobs_per_worker max_memory_mb max_worker_lifetime].freeze
595
+
596
+ # How many identical worker processes the supervisor forks for one
597
+ # capsule (issue #503). Default 1.
598
+ def processes_for(entry)
599
+ entry.fetch(:processes, nil) || 1
600
+ end
601
+
602
+ # One recycle limit for a capsule, falling back to the global value
603
+ # (mirrors read_ahead_for). nil means the limit is off.
604
+ def recycle_limit_for(entry, key)
605
+ entry.fetch(key, nil) || public_send(key)
606
+ end
607
+
576
608
  # Returns the execution mode for a specific worker config hash,
577
609
  # falling back to the global execution_mode setting.
578
610
  def execution_mode_for(worker_config)
@@ -828,6 +860,9 @@ module Pgbus
828
860
  "prefetch_limit must be > 0"
829
861
  end
830
862
 
863
+ validate_read_ahead!
864
+ validate_capsule_processes!
865
+
831
866
  if priority_levels && !(priority_levels.is_a?(Integer) && priority_levels >= 1 && priority_levels <= 10)
832
867
  raise Pgbus::ConfigurationError, "priority_levels must be an integer between 1 and 10"
833
868
  end
@@ -846,6 +881,54 @@ module Pgbus
846
881
  self
847
882
  end
848
883
 
884
+ # Global, per-capsule and per-consumer read_ahead: a non-negative Integer.
885
+ def validate_read_ahead!
886
+ settings = [["read_ahead", read_ahead]]
887
+ Array(workers).each { |w| settings << ["worker read_ahead", w[:read_ahead]] unless w[:read_ahead].nil? }
888
+ Array(event_consumers).each { |c| settings << ["event consumer read_ahead", c[:read_ahead]] unless c[:read_ahead].nil? }
889
+
890
+ settings.each do |name, value|
891
+ next if value.is_a?(Integer) && !value.negative?
892
+
893
+ raise Pgbus::ConfigurationError, "#{name} must be a non-negative Integer (got #{value.inspect})"
894
+ end
895
+ warn_read_ahead_without_heartbeat(settings)
896
+ end
897
+
898
+ # Per-capsule processes (a positive Integer) and recycle limits (a positive
899
+ # number; unset falls back to the global value). The Array form of
900
+ # +workers=+ bypasses +capsule+, so both are checked here (issue #503).
901
+ def validate_capsule_processes!
902
+ Array(workers).each do |w|
903
+ validate_processes!(w[:processes]) unless w[:processes].nil?
904
+ RECYCLE_LIMIT_KEYS.each do |key|
905
+ value = w[key]
906
+ next if value.nil? || (value.is_a?(Numeric) && value.positive?)
907
+
908
+ raise Pgbus::ConfigurationError, "capsule #{key} must be a positive number (got #{value.inspect})"
909
+ end
910
+ end
911
+ end
912
+
913
+ def validate_processes!(value)
914
+ return if value.is_a?(Integer) && value.positive?
915
+
916
+ raise Pgbus::ConfigurationError, "capsule processes must be a positive Integer (got #{value.inspect})"
917
+ end
918
+
919
+ # Buffered claims are kept invisible only by the visibility heartbeat.
920
+ # Without it a message buffered longer than visibility_timeout can be
921
+ # claimed by another worker while it still waits here. Legal, but warned.
922
+ def warn_read_ahead_without_heartbeat(settings)
923
+ return if visibility_heartbeat
924
+ return unless settings.any? { |_name, value| value.positive? }
925
+
926
+ Pgbus.logger.warn do
927
+ "[Pgbus] read_ahead is on but visibility_heartbeat is false — a message buffered longer than " \
928
+ "visibility_timeout (#{visibility_timeout}s) can be claimed and run by another worker"
929
+ end
930
+ end
931
+
849
932
  # An explicit shutdown_timeout must be a positive number; nil keeps the
850
933
  # derived drain_timeout + margin default. A value below drain_timeout is
851
934
  # legal but self-defeating (the supervisor SIGKILLs workers mid-drain), so
@@ -1163,15 +1246,23 @@ module Pgbus
1163
1246
  #
1164
1247
  # c.capsule :critical, queues: %w[critical], threads: 5
1165
1248
  # c.capsule :gated, queues: %w[gated], threads: 1, single_active_consumer: true
1249
+ # c.capsule :render, queues: %w[render], threads: 1, processes: 4,
1250
+ # max_memory_mb: 1_536, max_worker_lifetime: 6.hours
1251
+ #
1252
+ # processes: N forks N identical workers for the capsule (CPU-bound jobs:
1253
+ # threads share one GVL, processes do not). The capsule-level recycle
1254
+ # limits override the global ones for this capsule's workers only.
1166
1255
  #
1167
1256
  # Names must be unique. Queues must not overlap with capsules already
1168
1257
  # defined (would cause double-processing). Composes with the string DSL —
1169
1258
  # +c.workers "..."+ followed by +c.capsule :name, ...+ appends the
1170
1259
  # named capsule to the list parsed from the string.
1171
- def capsule(name, queues:, threads:, **)
1260
+ def capsule(name, queues:, threads:, **options)
1172
1261
  raise Pgbus::ConfigurationError, "capsule queues must be a non-empty Array" unless queues.is_a?(Array) && queues.any?
1173
1262
  raise Pgbus::ConfigurationError, "capsule threads must be a positive Integer" unless threads.is_a?(Integer) && threads.positive?
1174
1263
 
1264
+ validate_processes!(options[:processes]) unless options[:processes].nil?
1265
+
1175
1266
  normalized_name = name.to_s
1176
1267
  @workers ||= []
1177
1268
 
@@ -1181,7 +1272,7 @@ module Pgbus
1181
1272
 
1182
1273
  validate_no_queue_overlap!(queues)
1183
1274
 
1184
- @workers << { name: normalized_name, queues: queues, threads: threads, ** }
1275
+ @workers << { name: normalized_name, queues: queues, threads: threads, **options }
1185
1276
  end
1186
1277
 
1187
1278
  # Look up a capsule by its name. Accepts symbol or string. Returns the
@@ -0,0 +1,129 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "time"
5
+
6
+ module Pgbus
7
+ # The `pgbus_dead_letter` block a mover writes into a message's PGMQ
8
+ # headers when it moves the message to a dead-letter queue (issue #495):
9
+ # why it was moved, from where, after how many attempts, and the last
10
+ # recorded error. Pure — no I/O — so the executor, the consumer, the
11
+ # dashboard, MCP and the CLI all read and write the same shape.
12
+ #
13
+ # PGMQ headers travel as raw JSON strings, so every entry point accepts a
14
+ # String (or a Hash, or nil) and build/strip_for_retry return a String.
15
+ module DeadLetterHeader
16
+ KEY = "pgbus_dead_letter"
17
+ # Top-level counter on a live message retried out of a DLQ. Folded into
18
+ # the block as `retries_from_dlq` when the message dies again.
19
+ RETRIES_KEY = "pgbus_dlq_retries"
20
+ ORIGINAL_KEY = "pgbus_original_headers"
21
+ VERSION = 1
22
+ REASON_MAX_RETRIES = "max_retries_exceeded"
23
+ # The release that started writing the block: rows without it predate it.
24
+ SINCE = "0.18.0"
25
+ MESSAGE_LIMIT = 1_000
26
+ BACKTRACE_LINES = 10
27
+ BACKTRACE_LIMIT = 2_000
28
+
29
+ module_function
30
+
31
+ # error: FailedEventRecorder.last_error's Hash, or nil when no failure
32
+ # was recorded. Returns the merged headers as a JSON string.
33
+ def build(existing:, reason:, source:, source_queue:, attempts:, max_retries:, error: nil, now: Time.now.utc)
34
+ headers = decode(existing, warn: true)
35
+ retries = count(headers.delete(RETRIES_KEY))
36
+ block = {
37
+ "version" => VERSION,
38
+ "reason" => reason,
39
+ "source" => source,
40
+ "source_queue" => source_queue,
41
+ "attempts" => attempts,
42
+ "max_retries" => max_retries,
43
+ "dead_lettered_at" => iso(now)
44
+ }
45
+ block["retries_from_dlq"] = retries if retries.positive?
46
+ block.merge!(error_fields(error)) if error
47
+ JSON.generate(headers.merge(KEY => block))
48
+ end
49
+
50
+ # The block as a string-keyed Hash, or nil. Never raises.
51
+ def parse(headers)
52
+ block = decode(headers, warn: false)[KEY]
53
+ block.is_a?(Hash) ? block : nil
54
+ end
55
+
56
+ # A live message must not claim to be dead: drop the block, and carry
57
+ # how many times it has come back out of a DLQ. A DLQ message has no
58
+ # top-level counter (build folded it into the block), so the block is
59
+ # the only place to read the previous count from.
60
+ def strip_for_retry(headers)
61
+ retries = count(parse(headers)&.dig("retries_from_dlq")) + 1
62
+ stripped = decode(headers, warn: false)
63
+ stripped.delete(KEY)
64
+ stripped[RETRIES_KEY] = retries
65
+ JSON.generate(stripped)
66
+ end
67
+
68
+ # A counter read from headers anyone could have written: an Integer or a
69
+ # numeric String counts, anything else is 0. Never raises, so a bad value
70
+ # can't stop a dead-letter.
71
+ def count(value)
72
+ case value
73
+ when Integer then value
74
+ when String then Integer(value, 10, exception: false) || 0
75
+ else 0
76
+ end
77
+ end
78
+
79
+ def error_fields(error)
80
+ {
81
+ "error_class" => error[:error_class],
82
+ "error_message" => error[:error_message]&.to_s&.slice(0, MESSAGE_LIMIT),
83
+ "backtrace" => backtrace_head(error[:backtrace]),
84
+ "error_attempt" => error[:retry_count].to_i + 1,
85
+ "error_recorded_at" => error[:failed_at] && iso(error[:failed_at])
86
+ }.compact
87
+ end
88
+
89
+ def backtrace_head(backtrace)
90
+ return if backtrace.nil? || backtrace.empty?
91
+
92
+ lines = backtrace.is_a?(Array) ? backtrace : backtrace.to_s.split("\n")
93
+ lines.first(BACKTRACE_LINES).join("\n").slice(0, BACKTRACE_LIMIT)
94
+ end
95
+
96
+ def iso(value)
97
+ case value
98
+ when Time then value.getutc.iso8601(6)
99
+ when String then value
100
+ else value.respond_to?(:to_time) ? value.to_time.utc.iso8601(6) : value.to_s
101
+ end
102
+ end
103
+
104
+ # Always a Hash. Headers that are not a JSON object are kept, not
105
+ # dropped, under ORIGINAL_KEY.
106
+ def decode(headers, warn:)
107
+ case headers
108
+ when nil, "" then {}
109
+ when Hash then headers.transform_keys(&:to_s)
110
+ else wrap(JSON.parse(headers.to_s), headers, warn)
111
+ end
112
+ rescue JSON::ParserError
113
+ log_kept(headers) if warn
114
+ { ORIGINAL_KEY => headers }
115
+ end
116
+
117
+ def wrap(parsed, raw, warn)
118
+ return parsed if parsed.is_a?(Hash)
119
+
120
+ log_kept(raw) if warn
121
+ { ORIGINAL_KEY => parsed }
122
+ end
123
+
124
+ # The raw value is not logged: headers can carry caller secrets.
125
+ def log_kept(_raw)
126
+ Pgbus.logger.warn { "[Pgbus] Dead-letter headers were not a JSON object; kept under #{ORIGINAL_KEY}" }
127
+ end
128
+ end
129
+ end
data/lib/pgbus/doctor.rb CHANGED
@@ -5,7 +5,7 @@ require "pgbus/mcp/health_analyzer"
5
5
 
6
6
  module Pgbus
7
7
  # Preflight diagnostics for a pgbus deployment — the single command that
8
- # answers "is this environment healthy enough to run?". Runs eleven checks and
8
+ # answers "is this environment healthy enough to run?". Runs twelve checks and
9
9
  # returns a machine-readable result plus a human report, so `pgbus doctor`
10
10
  # and `rake pgbus:doctor` can gate a deploy or CI run (exit 0 on success,
11
11
  # 1 on any failure).
@@ -32,6 +32,7 @@ module Pgbus
32
32
  # stable identity of each check (used in the report and to select subsets).
33
33
  CHECKS = {
34
34
  "Configuration" => :check_configuration,
35
+ "Ruby JIT" => :check_ruby_jit,
35
36
  "Database" => :check_database,
36
37
  "PGMQ schema" => :check_pgmq_schema,
37
38
  "Queues" => :check_queues,
@@ -166,6 +167,33 @@ module Pgbus
166
167
  Check.new(name: "Configuration", status: :fail, detail: "#{e.class}: #{e.message}")
167
168
  end
168
169
 
170
+ # 1b. Ruby JIT (issue #484). With Postgres close by, a worker is GVL-bound
171
+ # and YJIT is worth ~15-20% jobs/s (docs/performance.md). Rails enables it via
172
+ # config.yjit (load_defaults 7.2+, or 8.0+ outside local envs). Reports
173
+ # only — enabling a JIT is the application's decision, never pgbus's.
174
+ def check_ruby_jit
175
+ label = RubyJit.label
176
+ return Check.new(name: "Ruby JIT", status: :ok, detail: "#{label.upcase} enabled") unless label == "none"
177
+ return Check.new(name: "Ruby JIT", status: :ok, detail: "no YJIT available on this Ruby") unless RubyJit.yjit_available?
178
+ return Check.new(name: "Ruby JIT", status: :ok, detail: "YJIT off in a local environment (expected)") if local_env?
179
+
180
+ Check.new(name: "Ruby JIT", status: :warn,
181
+ detail: "YJIT is available but off — a CPU-bound worker (Postgres nearby) runs ~15% fewer jobs/s " \
182
+ "without it; no measurable effect when network-bound. " \
183
+ "Enable it with config.yjit = true (or load_defaults 7.2+) or RUBY_YJIT_ENABLE=1")
184
+ rescue StandardError => e
185
+ Check.new(name: "Ruby JIT", status: :warn, detail: "could not determine (#{e.class}: #{e.message})")
186
+ end
187
+
188
+ # A Rails app outside development/test. Without Rails the environment is
189
+ # unknown, so assume local and stay quiet rather than warn on a guess.
190
+ def local_env?
191
+ return true unless defined?(Rails) && Rails.respond_to?(:env) && Rails.env
192
+
193
+ env = Rails.env
194
+ env.respond_to?(:local?) ? env.local? : %w[development test].include?(env.to_s)
195
+ end
196
+
169
197
  # 2. Database connectivity — SELECT 1 via Client#ping.
170
198
  def check_database
171
199
  @client.ping
@@ -374,23 +402,27 @@ module Pgbus
374
402
  # per worker/consumer fork. Streams add one per web-server process, which
375
403
  # the doctor cannot count from here, so it is reported as a clause.
376
404
  def check_connection_budget
377
- capsules = @config.role_enabled?(:workers) ? Array(@config.workers).size : 0
405
+ worker_configs = @config.role_enabled?(:workers) ? Array(@config.workers) : []
406
+ capsules = worker_configs.size
407
+ # A processes: N capsule forks N workers, each with its own LISTEN under :fork scope.
408
+ worker_forks = worker_configs.sum { |w| @config.processes_for(w) }
378
409
  consumers = @config.role_enabled?(:consumers) ? Array(@config.event_consumers).size : 0
379
410
 
380
411
  count =
381
412
  if !@config.worker_notify_wakeup?
382
413
  0
383
414
  elsif @config.worker_notify_scope == :supervisor
384
- (capsules + consumers).positive? ? 1 : 0
415
+ (worker_forks + consumers).positive? ? 1 : 0
385
416
  else
386
- capsules + consumers
417
+ worker_forks + consumers
387
418
  end
388
419
 
389
420
  detail = format(
390
421
  "%<count>d direct LISTEN connection%<plural>s pinned (scope=%<scope>s; " \
391
- "%<capsules>d capsule%<cap_plural>s + %<consumers>d consumer%<con_plural>s%<share>s)",
422
+ "%<capsules>d capsule%<cap_plural>s%<forks>s + %<consumers>d consumer%<con_plural>s%<share>s)",
392
423
  count: count, plural: count == 1 ? "" : "s", scope: @config.worker_notify_scope,
393
424
  capsules: capsules, cap_plural: capsules == 1 ? "" : "s",
425
+ forks: worker_forks == capsules ? "" : " (#{worker_forks} worker processes)",
394
426
  consumers: consumers, con_plural: consumers == 1 ? "" : "s",
395
427
  share: count == 1 && @config.worker_notify_scope == :supervisor ? " share it" : ""
396
428
  )
@@ -15,6 +15,23 @@ module Pgbus
15
15
  def dedup_cache
16
16
  @dedup_cache ||= DedupCache.new
17
17
  end
18
+
19
+ # How long a pending claim may stay silent before its holder counts as
20
+ # dead. ClaimBeat refreshes a live claim from the visibility heartbeat,
21
+ # which lands every extension inside [interval, 1.5 * interval] of the
22
+ # previous one — two intervals leaves margin for a late beat without
23
+ # stretching the window past the visibility timeout it rides on.
24
+ #
25
+ # With the heartbeat disabled a claim is never refreshed, so the window
26
+ # degrades to "roughly the first two thirds of one visibility timeout
27
+ # after the claim" — a redelivery, which cannot arrive before the VT has
28
+ # lapsed, still re-runs exactly as it did before issue #470.
29
+ #
30
+ # The dashboard reads the same window to tell a claim being handled
31
+ # from one that went silent (issue #494).
32
+ def claim_ownership_window
33
+ Pgbus.configuration.effective_visibility_heartbeat_interval * 2
34
+ end
18
35
  end
19
36
 
20
37
  # Outcome of the two-phase claim. `age` is how long the losing delivery
@@ -196,18 +213,8 @@ module Pgbus
196
213
  ClaimResult.new(status: :claimed, age: age)
197
214
  end
198
215
 
199
- # How long a pending claim may stay silent before its holder counts as
200
- # dead. ClaimBeat refreshes a live claim from the visibility heartbeat,
201
- # which lands every extension inside [interval, 1.5 * interval] of the
202
- # previous one — two intervals leaves margin for a late beat without
203
- # stretching the window past the visibility timeout it rides on.
204
- #
205
- # With the heartbeat disabled a claim is never refreshed, so the window
206
- # degrades to "roughly the first two thirds of one visibility timeout
207
- # after the claim" — a redelivery, which cannot arrive before the VT has
208
- # lapsed, still re-runs exactly as it did before issue #470.
209
216
  def claim_ownership_window
210
- Pgbus.configuration.effective_visibility_heartbeat_interval * 2
217
+ self.class.claim_ownership_window
211
218
  end
212
219
 
213
220
  # Register this claim with the message's beat for exactly the duration of
@@ -125,6 +125,12 @@ module Pgbus
125
125
  subscription_pattern.start_with?(topic_filter.delete_suffix(".#"))
126
126
  end
127
127
 
128
+ # Whether +routing_key+ matches a subscription +pattern+ ("*" one word,
129
+ # "#" any). The dashboard checks a replay with it (issue #494).
130
+ def pattern_matches?(pattern, routing_key)
131
+ matches?(pattern, routing_key)
132
+ end
133
+
128
134
  private
129
135
 
130
136
  # True when running inside a rake schema/asset task, where setup_all!(safe:)