pgbus 0.17.1 → 0.17.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +42 -0
  3. data/README.md +83 -13
  4. data/Rakefile +26 -0
  5. data/app/assets/javascripts/pgbus/stream_source_element.js +34 -2
  6. data/app/assets/javascripts/pgbus/stream_source_element.min.js +6 -0
  7. data/app/assets/javascripts/pgbus/stream_source_element.min.js.map +1 -0
  8. data/app/controllers/concerns/pgbus/job_listing.rb +37 -0
  9. data/app/controllers/concerns/pgbus/process_listing.rb +17 -0
  10. data/app/controllers/pgbus/application_controller.rb +5 -2
  11. data/app/controllers/pgbus/batches_controller.rb +4 -1
  12. data/app/controllers/pgbus/dashboard_controller.rb +20 -6
  13. data/app/controllers/pgbus/dead_letter_controller.rb +27 -3
  14. data/app/controllers/pgbus/events_controller.rb +52 -3
  15. data/app/controllers/pgbus/frontends_controller.rb +25 -11
  16. data/app/controllers/pgbus/insights_controller.rb +19 -1
  17. data/app/controllers/pgbus/jobs_controller.rb +45 -26
  18. data/app/controllers/pgbus/locks_controller.rb +7 -2
  19. data/app/controllers/pgbus/outbox_controller.rb +4 -1
  20. data/app/controllers/pgbus/processes_controller.rb +3 -1
  21. data/app/controllers/pgbus/queues_controller.rb +16 -4
  22. data/app/controllers/pgbus/recurring_tasks_controller.rb +5 -7
  23. data/app/frontend/pgbus/application.min.js +2 -0
  24. data/app/frontend/pgbus/application.min.js.map +1 -0
  25. data/app/frontend/pgbus/modules/charts.js +137 -77
  26. data/app/frontend/pgbus/modules/charts.min.js +2 -0
  27. data/app/frontend/pgbus/modules/charts.min.js.map +1 -0
  28. data/app/frontend/pgbus/style.css +1 -1
  29. data/app/frontend/pgbus/tailwind.css +18 -0
  30. data/app/frontend/pgbus/vendor/turbo.min.js +54 -0
  31. data/app/frontend/pgbus/vendor/turbo.min.js.map +1 -0
  32. data/app/helpers/pgbus/application_helper.rb +155 -113
  33. data/app/helpers/pgbus/button_helper.rb +88 -0
  34. data/app/helpers/pgbus/dashboard_helper.rb +157 -0
  35. data/app/helpers/pgbus/events_helper.rb +40 -0
  36. data/app/helpers/pgbus/insights_helper.rb +39 -0
  37. data/app/helpers/pgbus/processes_helper.rb +79 -0
  38. data/app/views/layouts/pgbus/application.html.erb +0 -1
  39. data/app/views/pgbus/batches/_batches_table.html.erb +16 -13
  40. data/app/views/pgbus/batches/show.html.erb +3 -4
  41. data/app/views/pgbus/dashboard/_processes_table.html.erb +11 -11
  42. data/app/views/pgbus/dashboard/_queue_health.html.erb +42 -51
  43. data/app/views/pgbus/dashboard/_queues_table.html.erb +18 -16
  44. data/app/views/pgbus/dashboard/_recent_failures.html.erb +16 -12
  45. data/app/views/pgbus/dashboard/_stats_cards.html.erb +70 -36
  46. data/app/views/pgbus/dashboard/show.html.erb +1 -1
  47. data/app/views/pgbus/dead_letter/_error.html.erb +13 -0
  48. data/app/views/pgbus/dead_letter/_messages_table.html.erb +37 -96
  49. data/app/views/pgbus/dead_letter/_row.html.erb +75 -0
  50. data/app/views/pgbus/dead_letter/index.html.erb +6 -9
  51. data/app/views/pgbus/dead_letter/show.html.erb +37 -21
  52. data/app/views/pgbus/events/_list.html.erb +55 -0
  53. data/app/views/pgbus/events/_processed_table.html.erb +45 -0
  54. data/app/views/pgbus/events/_replay.html.erb +10 -0
  55. data/app/views/pgbus/events/_row.html.erb +162 -0
  56. data/app/views/pgbus/events/_tabs.html.erb +13 -0
  57. data/app/views/pgbus/events/index.html.erb +27 -48
  58. data/app/views/pgbus/events/show.html.erb +25 -7
  59. data/app/views/pgbus/insights/_chart_card.html.erb +21 -0
  60. data/app/views/pgbus/insights/_latency_table.html.erb +20 -0
  61. data/app/views/pgbus/insights/_status_table.html.erb +19 -0
  62. data/app/views/pgbus/insights/_throughput_table.html.erb +18 -0
  63. data/app/views/pgbus/insights/show.html.erb +83 -100
  64. data/app/views/pgbus/jobs/_list.html.erb +64 -0
  65. data/app/views/pgbus/jobs/_row.html.erb +107 -0
  66. data/app/views/pgbus/jobs/_tabs.html.erb +14 -0
  67. data/app/views/pgbus/jobs/index.html.erb +10 -10
  68. data/app/views/pgbus/jobs/show.html.erb +18 -20
  69. data/app/views/pgbus/locks/_concurrency.html.erb +25 -23
  70. data/app/views/pgbus/locks/_uniqueness.html.erb +20 -22
  71. data/app/views/pgbus/outbox/index.html.erb +30 -27
  72. data/app/views/pgbus/processes/_processes_table.html.erb +6 -25
  73. data/app/views/pgbus/processes/_row.html.erb +48 -0
  74. data/app/views/pgbus/processes/_status.html.erb +4 -0
  75. data/app/views/pgbus/queues/_queues_list.html.erb +28 -32
  76. data/app/views/pgbus/queues/show.html.erb +69 -118
  77. data/app/views/pgbus/recurring_tasks/_tasks_table.html.erb +75 -76
  78. data/app/views/pgbus/recurring_tasks/index.html.erb +1 -1
  79. data/app/views/pgbus/recurring_tasks/show.html.erb +45 -51
  80. data/app/views/pgbus/shared/_empty_row.html.erb +2 -0
  81. data/app/views/pgbus/shared/_pager.html.erb +34 -0
  82. data/config/i18n-tasks.yml +11 -2
  83. data/config/locales/da.yml +405 -113
  84. data/config/locales/de.yml +405 -113
  85. data/config/locales/en.yml +405 -113
  86. data/config/locales/es.yml +404 -112
  87. data/config/locales/fi.yml +404 -112
  88. data/config/locales/fr.yml +406 -114
  89. data/config/locales/it.yml +405 -113
  90. data/config/locales/ja.yml +405 -113
  91. data/config/locales/nb.yml +405 -113
  92. data/config/locales/nl.yml +405 -113
  93. data/config/locales/pt.yml +406 -114
  94. data/config/locales/sv.yml +404 -112
  95. data/config/routes.rb +1 -1
  96. data/lib/generators/pgbus/add_presence_generator.rb +2 -1
  97. data/lib/generators/pgbus/add_presence_seats_generator.rb +45 -0
  98. data/lib/generators/pgbus/templates/add_presence.rb.erb +32 -7
  99. data/lib/generators/pgbus/templates/add_presence_seats.rb.erb +38 -0
  100. data/lib/generators/pgbus/templates/initializer.rb.erb +4 -0
  101. data/lib/pgbus/active_job/executor.rb +16 -6
  102. data/lib/pgbus/cli/dlq.rb +32 -5
  103. data/lib/pgbus/client.rb +4 -2
  104. data/lib/pgbus/configuration/presence_settings.rb +103 -0
  105. data/lib/pgbus/configuration.rb +51 -2
  106. data/lib/pgbus/dead_letter_header.rb +129 -0
  107. data/lib/pgbus/doctor.rb +8 -4
  108. data/lib/pgbus/engine.rb +13 -7
  109. data/lib/pgbus/event_bus/handler.rb +18 -11
  110. data/lib/pgbus/event_bus/registry.rb +6 -0
  111. data/lib/pgbus/failed_event_recorder.rb +34 -0
  112. data/lib/pgbus/generators/migration_detector.rb +9 -1
  113. data/lib/pgbus/instrumentation.rb +2 -1
  114. data/lib/pgbus/mcp/base_tool.rb +25 -0
  115. data/lib/pgbus/mcp/tools/concurrency_tool.rb +23 -4
  116. data/lib/pgbus/mcp/tools/dlq_detail_tool.rb +4 -2
  117. data/lib/pgbus/mcp/tools/dlq_tool.rb +10 -4
  118. data/lib/pgbus/mcp/tools/locks_tool.rb +23 -5
  119. data/lib/pgbus/mcp/tools/recurring_tool.rb +33 -4
  120. data/lib/pgbus/outbox/poller.rb +1 -1
  121. data/lib/pgbus/process/consumer.rb +53 -3
  122. data/lib/pgbus/process/dispatcher.rb +1 -1
  123. data/lib/pgbus/process/supervisor.rb +32 -11
  124. data/lib/pgbus/process/worker.rb +45 -8
  125. data/lib/pgbus/recurring/scheduler.rb +2 -1
  126. data/lib/pgbus/streams/presence.rb +317 -81
  127. data/lib/pgbus/version.rb +1 -1
  128. data/lib/pgbus/web/data_source/dashboard_signals.rb +170 -0
  129. data/lib/pgbus/web/data_source/dead_letter.rb +142 -0
  130. data/lib/pgbus/web/data_source/event_list.rb +161 -0
  131. data/lib/pgbus/web/data_source/job_list.rb +323 -0
  132. data/lib/pgbus/web/data_source/list_counts.rb +80 -0
  133. data/lib/pgbus/web/data_source/process_list.rb +69 -0
  134. data/lib/pgbus/web/data_source/queue_summary.rb +122 -0
  135. data/lib/pgbus/web/data_source.rb +132 -254
  136. data/lib/pgbus/web/dead_letter_reason.rb +81 -0
  137. data/lib/pgbus/web/event_state.rb +71 -0
  138. data/lib/pgbus/web/health_app.rb +7 -3
  139. data/lib/pgbus/web/health_verdict.rb +78 -0
  140. data/lib/pgbus/web/insights_charts.rb +84 -0
  141. data/lib/pgbus/web/job_state.rb +111 -0
  142. data/lib/pgbus/web/process_activity.rb +97 -0
  143. data/lib/pgbus/web/process_state.rb +187 -0
  144. data/lib/pgbus/web/queue_summary.rb +104 -0
  145. data/lib/pgbus/web/stream_app.rb +18 -6
  146. data/lib/pgbus/web/streamer/connection.rb +7 -1
  147. data/lib/pgbus/web/streamer/falcon_connection.rb +7 -1
  148. data/lib/pgbus/web/streamer/heartbeat.rb +9 -3
  149. data/lib/pgbus/web/streamer/instance.rb +2 -1
  150. data/lib/pgbus/web/streamer/presence_tracker.rb +155 -0
  151. data/lib/pgbus/web/streamer/stream_event_dispatcher.rb +8 -37
  152. data/lib/pgbus/web/time_format.rb +128 -0
  153. metadata +55 -4
  154. data/app/views/pgbus/events/_pending_table.html.erb +0 -169
  155. data/app/views/pgbus/jobs/_enqueued_table.html.erb +0 -118
  156. data/app/views/pgbus/jobs/_failed_table.html.erb +0 -76
data/lib/pgbus/doctor.rb CHANGED
@@ -402,23 +402,27 @@ module Pgbus
402
402
  # per worker/consumer fork. Streams add one per web-server process, which
403
403
  # the doctor cannot count from here, so it is reported as a clause.
404
404
  def check_connection_budget
405
- capsules = @config.role_enabled?(:workers) ? Array(@config.workers).size : 0
405
+ worker_configs = @config.role_enabled?(:workers) ? Array(@config.workers) : []
406
+ capsules = worker_configs.size
407
+ # A processes: N capsule forks N workers, each with its own LISTEN under :fork scope.
408
+ worker_forks = worker_configs.sum { |w| @config.processes_for(w) }
406
409
  consumers = @config.role_enabled?(:consumers) ? Array(@config.event_consumers).size : 0
407
410
 
408
411
  count =
409
412
  if !@config.worker_notify_wakeup?
410
413
  0
411
414
  elsif @config.worker_notify_scope == :supervisor
412
- (capsules + consumers).positive? ? 1 : 0
415
+ (worker_forks + consumers).positive? ? 1 : 0
413
416
  else
414
- capsules + consumers
417
+ worker_forks + consumers
415
418
  end
416
419
 
417
420
  detail = format(
418
421
  "%<count>d direct LISTEN connection%<plural>s pinned (scope=%<scope>s; " \
419
- "%<capsules>d capsule%<cap_plural>s + %<consumers>d consumer%<con_plural>s%<share>s)",
422
+ "%<capsules>d capsule%<cap_plural>s%<forks>s + %<consumers>d consumer%<con_plural>s%<share>s)",
420
423
  count: count, plural: count == 1 ? "" : "s", scope: @config.worker_notify_scope,
421
424
  capsules: capsules, cap_plural: capsules == 1 ? "" : "s",
425
+ forks: worker_forks == capsules ? "" : " (#{worker_forks} worker processes)",
422
426
  consumers: consumers, con_plural: consumers == 1 ? "" : "s",
423
427
  share: count == 1 && @config.worker_notify_scope == :supervisor ? " share it" : ""
424
428
  )
data/lib/pgbus/engine.rb CHANGED
@@ -139,14 +139,20 @@ module Pgbus
139
139
  app.middleware.use Pgbus::Streams::WatermarkCacheMiddleware if Pgbus.configuration.streams_enabled
140
140
  end
141
141
 
142
- # Make stream_source_element.js available to the host app's asset
143
- # pipeline (Propshaft or Sprockets) so it can be included via
144
- # `javascript_include_tag "pgbus/stream_source_element"` or pinned
145
- # in importmap. When importmap-rails is loaded, auto-pin it so
146
- # host apps get it without manual configuration.
142
+ # Make the <pgbus-stream-source> element available to the host app's
143
+ # asset pipeline (Propshaft or Sprockets). The importmap pin points at
144
+ # the minified build (rake frontend:js). Propshaft rewrites its
145
+ # sourceMappingURL to the digested .min.js.map; Sprockets serves the
146
+ # comment as written. The readable ES module source stays precompiled for
147
+ # apps that pin or bundle it themselves. When importmap-rails is loaded,
148
+ # auto-pin it so host apps get it without manual configuration.
147
149
  initializer "pgbus.streams.assets" do |app|
148
150
  if Pgbus.configuration.streams_enabled && app.config.respond_to?(:assets)
149
- app.config.assets.precompile += %w[pgbus/stream_source_element.js]
151
+ app.config.assets.precompile += %w[
152
+ pgbus/stream_source_element.min.js
153
+ pgbus/stream_source_element.min.js.map
154
+ pgbus/stream_source_element.js
155
+ ]
150
156
  end
151
157
  end
152
158
 
@@ -156,7 +162,7 @@ module Pgbus
156
162
  next unless Rails.application.respond_to?(:importmap)
157
163
 
158
164
  Rails.application.importmap.pin(
159
- "pgbus/stream_source_element", to: "pgbus/stream_source_element.js"
165
+ "pgbus/stream_source_element", to: "pgbus/stream_source_element.min.js"
160
166
  )
161
167
  end
162
168
  end
@@ -15,6 +15,23 @@ module Pgbus
15
15
  def dedup_cache
16
16
  @dedup_cache ||= DedupCache.new
17
17
  end
18
+
19
+ # How long a pending claim may stay silent before its holder counts as
20
+ # dead. ClaimBeat refreshes a live claim from the visibility heartbeat,
21
+ # which lands every extension inside [interval, 1.5 * interval] of the
22
+ # previous one — two intervals leaves margin for a late beat without
23
+ # stretching the window past the visibility timeout it rides on.
24
+ #
25
+ # With the heartbeat disabled a claim is never refreshed, so the window
26
+ # degrades to "roughly the first two thirds of one visibility timeout
27
+ # after the claim" — a redelivery, which cannot arrive before the VT has
28
+ # lapsed, still re-runs exactly as it did before issue #470.
29
+ #
30
+ # The dashboard reads the same window to tell a claim being handled
31
+ # from one that went silent (issue #494).
32
+ def claim_ownership_window
33
+ Pgbus.configuration.effective_visibility_heartbeat_interval * 2
34
+ end
18
35
  end
19
36
 
20
37
  # Outcome of the two-phase claim. `age` is how long the losing delivery
@@ -196,18 +213,8 @@ module Pgbus
196
213
  ClaimResult.new(status: :claimed, age: age)
197
214
  end
198
215
 
199
- # How long a pending claim may stay silent before its holder counts as
200
- # dead. ClaimBeat refreshes a live claim from the visibility heartbeat,
201
- # which lands every extension inside [interval, 1.5 * interval] of the
202
- # previous one — two intervals leaves margin for a late beat without
203
- # stretching the window past the visibility timeout it rides on.
204
- #
205
- # With the heartbeat disabled a claim is never refreshed, so the window
206
- # degrades to "roughly the first two thirds of one visibility timeout
207
- # after the claim" — a redelivery, which cannot arrive before the VT has
208
- # lapsed, still re-runs exactly as it did before issue #470.
209
216
  def claim_ownership_window
210
- Pgbus.configuration.effective_visibility_heartbeat_interval * 2
217
+ self.class.claim_ownership_window
211
218
  end
212
219
 
213
220
  # Register this claim with the message's beat for exactly the duration of
@@ -125,6 +125,12 @@ module Pgbus
125
125
  subscription_pattern.start_with?(topic_filter.delete_suffix(".#"))
126
126
  end
127
127
 
128
+ # Whether +routing_key+ matches a subscription +pattern+ ("*" one word,
129
+ # "#" any). The dashboard checks a replay with it (issue #494).
130
+ def pattern_matches?(pattern, routing_key)
131
+ matches?(pattern, routing_key)
132
+ end
133
+
128
134
  private
129
135
 
130
136
  # True when running inside a rake schema/asset task, where setup_all!(safe:)
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "time"
4
+
3
5
  module Pgbus
4
6
  # Records job failures to pgbus_failed_events for dashboard visibility.
5
7
  # Uses upsert (INSERT ON CONFLICT UPDATE) keyed on (queue_name, msg_id)
@@ -47,6 +49,31 @@ module Pgbus
47
49
  false
48
50
  end
49
51
 
52
+ # The message's latest recorded failure, read by a mover just before it
53
+ # dead-letters the message and clears the row (issue #495). failed_at
54
+ # comes back as an ISO 8601 UTC string whatever the adapter decodes it
55
+ # to. A failed read returns nil: it must never stop a dead-letter.
56
+ def last_error(queue_name:, msg_id:)
57
+ row = connection.select_one(
58
+ "SELECT error_class, error_message, backtrace, retry_count, failed_at " \
59
+ "FROM pgbus_failed_events WHERE queue_name = $1 AND msg_id = $2 LIMIT 1",
60
+ "FailedEvent Last Error",
61
+ [queue_name, msg_id.to_i]
62
+ )
63
+ return unless row
64
+
65
+ {
66
+ error_class: row["error_class"],
67
+ error_message: row["error_message"],
68
+ backtrace: row["backtrace"],
69
+ retry_count: row["retry_count"].to_i,
70
+ failed_at: iso_utc(row["failed_at"])
71
+ }
72
+ rescue StandardError => e
73
+ Pgbus.logger.debug { "[Pgbus] FailedEvent last_error read failed: #{e.class}: #{e.message}" }
74
+ nil
75
+ end
76
+
50
77
  # Drops every row recorded under any of `queue_names`. Called when a
51
78
  # queue is dropped: PGMQ restarts msg_ids on recreate, so a surviving
52
79
  # row would aim the dashboard's retry/discard at an unrelated message.
@@ -82,6 +109,13 @@ module Pgbus
82
109
 
83
110
  private
84
111
 
112
+ def iso_utc(value)
113
+ time = value.is_a?(String) ? Time.parse(value) : value
114
+ time&.utc&.iso8601(6)
115
+ rescue ArgumentError
116
+ value.to_s
117
+ end
118
+
85
119
  def connection
86
120
  if defined?(BusRecord) && BusRecord.connected?
87
121
  BusRecord.connection
@@ -73,6 +73,7 @@ module Pgbus
73
73
  add_stream_stats: "pgbus:add_stream_stats",
74
74
  add_stream_queues: "pgbus:add_stream_queues",
75
75
  add_presence: "pgbus:add_presence",
76
+ add_presence_seats: "pgbus:add_presence_seats",
76
77
  add_queue_states: "pgbus:add_queue_states",
77
78
  add_outbox: "pgbus:add_outbox",
78
79
  add_recurring: "pgbus:add_recurring",
@@ -94,6 +95,7 @@ module Pgbus
94
95
  add_stream_stats: "stream stats table (opt-in real-time Insights)",
95
96
  add_stream_queues: "stream queue registry (per-stream retention, orphan sweep, wildcard-worker safety)",
96
97
  add_presence: "presence members table (Turbo Streams presence)",
98
+ add_presence_seats: "presence seats (one row per SSE connection, per-seat state)",
97
99
  add_queue_states: "queue states table (pause/resume)",
98
100
  add_outbox: "outbox entries table (transactional outbox)",
99
101
  add_recurring: "recurring tasks + executions tables",
@@ -182,8 +184,14 @@ module Pgbus
182
184
  table_exists?("pgbus_stream_queues") ? [] : [:add_stream_queues]
183
185
  end
184
186
 
187
+ # The add_presence template already carries the seat columns, so a
188
+ # missing table needs only that; an existing table without
189
+ # connection_id predates seats (issue #515).
185
190
  def presence_migrations
186
- table_exists?("pgbus_presence_members") ? [] : [:add_presence]
191
+ return [:add_presence] unless table_exists?("pgbus_presence_members")
192
+ return [] if column_names("pgbus_presence_members").include?("connection_id")
193
+
194
+ [:add_presence_seats]
187
195
  end
188
196
 
189
197
  def queue_states_migrations
@@ -15,7 +15,8 @@ module Pgbus
15
15
  # pgbus.executor.execute — full job execution (deserialize + perform + archive)
16
16
  # pgbus.job_completed — job archived successfully
17
17
  # pgbus.job_failed — job raised; carries :exception_object
18
- # pgbus.job_dead_lettered — job exceeded max_retries and was DLQ-routed
18
+ # pgbus.job_dead_lettered — job exceeded max_retries and was DLQ-routed;
19
+ # :error is the last recorded error class (or nil)
19
20
  # pgbus.job_visibility_extended — heartbeat re-armed a running job's visibility timeout
20
21
  # payload: queue, job_class, msg_id, vt, extensions
21
22
  # pgbus.event_processed — event handler succeeded
@@ -70,6 +70,31 @@ module Pgbus
70
70
  ::MCP::Tool::Response.new([{ type: "text", text: JSON.generate(redacted) }])
71
71
  end
72
72
 
73
+ # Past the count cap the total is a lower bound, so only a full page
74
+ # can have another one behind it. At the page ceiling the next call
75
+ # would clamp back to this page, so there is no "more" to fetch.
76
+ def more_pages?(count, page:, per_page:, shown:)
77
+ return false if page >= self::MAX_PAGE
78
+
79
+ count.capped? ? shown == per_page : page * per_page < count.total
80
+ end
81
+
82
+ DEAD_LETTER_SUMMARY_KEYS = %w[reason source source_queue attempts max_retries dead_lettered_at
83
+ retries_from_dlq error_class error_attempt error_recorded_at].freeze
84
+ DEAD_LETTER_ERROR_KEYS = %w[error_message backtrace].freeze
85
+
86
+ # Why a DLQ row died, from its DeadLetterHeader block (issue #495), or
87
+ # nil for a row dead-lettered before the block existed. The error
88
+ # message and backtrace can quote payload values, and the Redactor
89
+ # does not know those keys, so they are added only with include_error.
90
+ def dead_letter_summary(row, include_error:)
91
+ block = Pgbus::DeadLetterHeader.parse(row[:headers])
92
+ return unless block
93
+
94
+ keys = include_error ? DEAD_LETTER_SUMMARY_KEYS + DEAD_LETTER_ERROR_KEYS : DEAD_LETTER_SUMMARY_KEYS
95
+ block.slice(*keys)
96
+ end
97
+
73
98
  # Wrap an error message as an MCP error response (isError: true) so the
74
99
  # client surfaces it as a tool failure rather than a normal result.
75
100
  def error_response(message)
@@ -17,14 +17,33 @@ module Pgbus
17
17
  their limit — plus up to 100 key rows with value/limit, lease expiry,
18
18
  whether the lease is still fresh, parked count and oldest wait. A key
19
19
  with a stale lease and a parked backlog is a stuck pipeline: its holder
20
- died before releasing the slot.
20
+ died before releasing the slot. The key rows are paginated (default
21
+ 100 per page, capped at 100); the response also carries page,
22
+ per_page, total (number of keys, capped at 10000) and has_more.
21
23
  DESC
22
24
 
23
- input_schema(properties: {}, required: [])
25
+ MAX_PER_PAGE = 100
26
+ MAX_PAGE = 1_000
24
27
 
25
- def self.call(server_context: nil)
28
+ input_schema(
29
+ properties: {
30
+ page: { type: "integer", description: "1-based page number (default 1, max 1000).", minimum: 1 },
31
+ per_page: { type: "integer", description: "Rows per page (default 100, max 100).", minimum: 1 }
32
+ },
33
+ required: []
34
+ )
35
+
36
+ def self.call(page: 1, per_page: MAX_PER_PAGE, server_context: nil)
26
37
  data_source = data_source_from(server_context)
27
- json_response(data_source.concurrency_stats)
38
+ per_page = per_page.to_i.clamp(1, MAX_PER_PAGE)
39
+ page = page.to_i.clamp(1, MAX_PAGE)
40
+ stats = data_source.concurrency_stats(page: page, per_page: per_page)
41
+ count = data_source.list_count(:concurrency_keys)
42
+
43
+ json_response(
44
+ stats.merge(page: page, per_page: per_page, total: count.total,
45
+ has_more: more_pages?(count, page: page, per_page: per_page, shown: stats[:keys].size))
46
+ )
28
47
  end
29
48
  end
30
49
  end
@@ -12,7 +12,7 @@ module Pgbus
12
12
  title "Pgbus Dead-Letter Detail"
13
13
  description <<~DESC
14
14
  Inspect one dead-letter message: read_ct, vt, enqueued_at, source
15
- queue. Pass the full physical DLQ queue name (e.g.
15
+ queue, and a dead_letter summary of why it died. Pass the full physical DLQ queue name (e.g.
16
16
  "pgbus_default_dlq") in +queue+ to disambiguate — msg_ids are only
17
17
  unique within a single DLQ table, so the same id can exist in more
18
18
  than one. Without +queue+ the tool scans every "_dlq" table and
@@ -48,7 +48,9 @@ module Pgbus
48
48
  return error_response("Dead-letter message #{msg_id} not found") unless detail
49
49
  return detail if detail.is_a?(::MCP::Tool::Response)
50
50
 
51
- json_response({ message: detail }, server_context: server_context, include_payloads: include_payloads)
51
+ summary = dead_letter_summary(detail, include_error: payloads_allowed?(server_context, include_payloads))
52
+ json_response({ message: detail.merge(dead_letter: summary) },
53
+ server_context: server_context, include_payloads: include_payloads)
52
54
  end
53
55
 
54
56
  # When the caller didn't specify a queue, scan every DLQ for the id and
@@ -10,9 +10,12 @@ module Pgbus
10
10
  title "Pgbus Dead-Letter Queue"
11
11
  description <<~DESC
12
12
  List messages sitting in dead-letter queues (queues whose name ends
13
- in "_dlq") with read_ct, vt and enqueued_at. Paginated (default 25,
14
- max 100). Message bodies and headers are redacted unless the server
15
- allows payloads and include_payloads is set.
13
+ in "_dlq") with read_ct, vt and enqueued_at, and a dead_letter
14
+ summary of why each died (reason, source queue, attempts, error
15
+ class; null for messages dead-lettered before pgbus recorded it).
16
+ Paginated (default 25, max 100). Message bodies, headers and the
17
+ error message/backtrace are redacted unless the server allows
18
+ payloads and include_payloads is set.
16
19
  DESC
17
20
 
18
21
  MAX_PER_PAGE = 100
@@ -34,7 +37,10 @@ module Pgbus
34
37
  data_source = data_source_from(server_context)
35
38
  per_page = per_page.to_i.clamp(1, MAX_PER_PAGE)
36
39
  page = page.to_i.clamp(1, MAX_PAGE)
37
- rows = data_source.dlq_messages(page: page, per_page: per_page)
40
+ include_error = payloads_allowed?(server_context, include_payloads)
41
+ rows = data_source.dlq_messages(page: page, per_page: per_page).map do |row|
42
+ row.merge(dead_letter: dead_letter_summary(row, include_error: include_error))
43
+ end
38
44
 
39
45
  json_response(
40
46
  { page: page, per_page: per_page, messages: rows },
@@ -12,15 +12,33 @@ module Pgbus
12
12
  description <<~DESC
13
13
  List active job uniqueness locks with their lock_key, queue_name,
14
14
  msg_id, and age in seconds. A lock that outlives its message
15
- indicates a leaked uniqueness key blocking re-enqueues. Returns the
16
- 100 most recent locks.
15
+ indicates a leaked uniqueness key blocking re-enqueues. Paginated
16
+ (default 100 per page, capped at 100); the response carries page,
17
+ per_page, total (capped at 10000) and has_more.
17
18
  DESC
18
19
 
19
- input_schema(properties: {}, required: [])
20
+ MAX_PER_PAGE = 100
21
+ MAX_PAGE = 1_000
20
22
 
21
- def self.call(server_context: nil)
23
+ input_schema(
24
+ properties: {
25
+ page: { type: "integer", description: "1-based page number (default 1, max 1000).", minimum: 1 },
26
+ per_page: { type: "integer", description: "Rows per page (default 100, max 100).", minimum: 1 }
27
+ },
28
+ required: []
29
+ )
30
+
31
+ def self.call(page: 1, per_page: MAX_PER_PAGE, server_context: nil)
22
32
  data_source = data_source_from(server_context)
23
- json_response({ locks: data_source.job_locks })
33
+ per_page = per_page.to_i.clamp(1, MAX_PER_PAGE)
34
+ page = page.to_i.clamp(1, MAX_PAGE)
35
+ locks = data_source.job_locks(page: page, per_page: per_page)
36
+ count = data_source.list_count(:job_locks)
37
+
38
+ json_response(
39
+ { locks: locks, page: page, per_page: per_page, total: count.total,
40
+ has_more: more_pages?(count, page: page, per_page: per_page, shown: locks.size) }
41
+ )
24
42
  end
25
43
  end
26
44
  end
@@ -12,15 +12,44 @@ module Pgbus
12
12
  description <<~DESC
13
13
  List recurring (cron) tasks with their schedule, human-readable
14
14
  schedule, queue, enabled state, last_run_at and next_run_at. Use this
15
- to confirm scheduled work is firing on time.
15
+ to confirm scheduled work is firing on time. Returns every task unless
16
+ page or per_page is given; then it is paginated (per_page defaults to
17
+ 100, capped at 100). The response carries page, per_page, total
18
+ (capped at 10000 when paginated; the exact row count when not) and
19
+ has_more; per_page is null when unpaginated.
16
20
  DESC
17
21
 
18
- input_schema(properties: {}, required: [])
22
+ MAX_PER_PAGE = 100
23
+ MAX_PAGE = 1_000
19
24
 
20
- def self.call(server_context: nil)
25
+ input_schema(
26
+ properties: {
27
+ page: { type: "integer", description: "1-based page number (default 1, max 1000).", minimum: 1 },
28
+ per_page: { type: "integer", description: "Rows per page (default 100 once paginating, max 100).", minimum: 1 }
29
+ },
30
+ required: []
31
+ )
32
+
33
+ def self.call(page: nil, per_page: nil, server_context: nil)
21
34
  data_source = data_source_from(server_context)
22
- json_response({ recurring_tasks: data_source.recurring_tasks })
35
+ return unpaged(data_source) if page.nil? && per_page.nil?
36
+
37
+ per_page = (per_page || MAX_PER_PAGE).to_i.clamp(1, MAX_PER_PAGE)
38
+ page = (page || 1).to_i.clamp(1, MAX_PAGE)
39
+ tasks = data_source.recurring_tasks(page: page, per_page: per_page)
40
+ count = data_source.list_count(:recurring_tasks)
41
+
42
+ json_response(
43
+ { recurring_tasks: tasks, page: page, per_page: per_page, total: count.total,
44
+ has_more: more_pages?(count, page: page, per_page: per_page, shown: tasks.size) }
45
+ )
46
+ end
47
+
48
+ def self.unpaged(data_source)
49
+ tasks = data_source.recurring_tasks
50
+ json_response({ recurring_tasks: tasks, page: 1, per_page: nil, total: tasks.size, has_more: false })
23
51
  end
52
+ private_class_method :unpaged
24
53
  end
25
54
  end
26
55
  end
@@ -139,7 +139,7 @@ module Pgbus
139
139
  def start_heartbeat
140
140
  @heartbeat = Process::Heartbeat.new(
141
141
  kind: "outbox_poller",
142
- metadata: { pid: ::Process.pid }
142
+ metadata: { pid: ::Process.pid, started_at: Time.now.to_f, supervisor_pid: ::Process.ppid }
143
143
  )
144
144
  @heartbeat.start
145
145
  end
@@ -77,6 +77,8 @@ module Pgbus
77
77
  @claim_buffer = ClaimBuffer.new(config: config)
78
78
  @loop_tick_at = Concurrent::AtomicReference.new(nil)
79
79
  @started_at_monotonic = started_at_monotonic || monotonic_now
80
+ # Wall clock for the Processes page's uptime (issue #493).
81
+ @started_at = Time.current
80
82
  @wake_signal = WakeSignal.new
81
83
  @pool = ExecutionPools.build(
82
84
  mode: @execution_mode,
@@ -246,7 +248,7 @@ module Pgbus
246
248
 
247
249
  if message.read_ct.to_i > config.max_retries
248
250
  Pgbus.logger.warn { "[Pgbus] Consumer moving message #{message.msg_id} to DLQ after #{message.read_ct} reads" }
249
- Pgbus.client.move_to_dead_letter(queue_name, message)
251
+ dead_letter(message, queue_name)
250
252
  record_stat(message, queue_name, "dead_lettered", execution_start)
251
253
  return
252
254
  end
@@ -263,6 +265,9 @@ module Pgbus
263
265
  end
264
266
 
265
267
  Pgbus.client.archive_message(queue_name, message.msg_id.to_i)
268
+ # A redelivery that succeeds leaves no stale failure behind. First
269
+ # deliveries never had one: no query on the hot path (#484's gate).
270
+ FailedEventRecorder.clear!(queue_name: queue_name, msg_id: message.msg_id.to_i) if message.read_ct.to_i > 1
266
271
  @circuit_breaker.record_success(queue_name)
267
272
  record_stat(message, queue_name, "success", execution_start)
268
273
  rescue StandardError => e
@@ -270,6 +275,9 @@ module Pgbus
270
275
  # Message stays in queue; VT will expire and it becomes available again.
271
276
  # read_ct tracks delivery attempts — when it exceeds max_retries,
272
277
  # the next read will route to DLQ above.
278
+ # A dead-letter read fails only in the move: keep the handler's error
279
+ # on the row for the next attempt's DLQ block instead of the move's.
280
+ record_failed_event(message, queue_name, e) unless message.read_ct.to_i > config.max_retries
273
281
  @circuit_breaker.record_failure(queue_name)
274
282
  record_stat(message, queue_name, "failed", execution_start)
275
283
  ensure
@@ -281,6 +289,31 @@ module Pgbus
281
289
  @jobs_processed.increment
282
290
  end
283
291
 
292
+ # Order: read the last error, move with the dead-letter block, then
293
+ # clear the failed-event row. A move that raises keeps the row, so the
294
+ # next attempt still carries the reason.
295
+ def dead_letter(message, queue_name)
296
+ last_error = FailedEventRecorder.last_error(queue_name: queue_name, msg_id: message.msg_id.to_i)
297
+ headers = DeadLetterHeader.build(
298
+ existing: message.headers, reason: DeadLetterHeader::REASON_MAX_RETRIES, source: "consumer",
299
+ source_queue: config.queue_name(queue_name), attempts: message.read_ct.to_i,
300
+ max_retries: config.max_retries, error: last_error
301
+ )
302
+ Pgbus.client.move_to_dead_letter(queue_name, message, headers: headers)
303
+ FailedEventRecorder.clear!(queue_name: queue_name, msg_id: message.msg_id.to_i)
304
+ end
305
+
306
+ # The executor's handle_failure, for events (issue #494): one upsert per
307
+ # failure, under the LOGICAL queue name, so the dashboard reads the event
308
+ # as retrying with its error. No ErrorReporter call: Handler#process!
309
+ # already instruments pgbus.event_failed, which feeds the reporters.
310
+ def record_failed_event(message, queue_name, error)
311
+ FailedEventRecorder.record!(
312
+ queue_name: queue_name, msg_id: message.msg_id.to_i, payload: message.message,
313
+ headers: message.headers, error: error, retry_count: [message.read_ct.to_i - 1, 0].max
314
+ )
315
+ end
316
+
284
317
  # Run every handler that owns this message, with the message's visibility
285
318
  # timeout held open for as long as they take (issue #470).
286
319
  #
@@ -582,13 +615,30 @@ module Pgbus
582
615
  def start_heartbeat
583
616
  @heartbeat = Heartbeat.new(
584
617
  kind: "consumer",
585
- metadata: { topics: topics, threads: threads, pid: ::Process.pid },
618
+ metadata: {
619
+ topics: topics, threads: threads, pid: ::Process.pid, execution_mode: @execution_mode,
620
+ started_at: @started_at.to_f, supervisor_pid: ::Process.ppid
621
+ },
586
622
  on_beat: -> { on_heartbeat },
587
- loop_tick_supplier: -> { @loop_tick_at.get }
623
+ loop_tick_supplier: -> { @loop_tick_at.get },
624
+ metadata_supplier: -> { throughput_metadata }
588
625
  )
589
626
  @heartbeat.start
590
627
  end
591
628
 
629
+ # What the Processes page states about a consumer (issue #493), read
630
+ # once per beat, never per message. String keys: the value round-trips
631
+ # through JSON in the metadata column.
632
+ def throughput_metadata
633
+ pool = @pool.metadata
634
+ {
635
+ "jobs_processed" => @jobs_processed.value, "buffered" => @claim_buffer.size,
636
+ "capacity" => pool[:capacity], "busy" => pool[:busy], "memory_mb" => current_memory_mb,
637
+ "max_jobs" => config.max_jobs_per_worker, "max_memory_mb" => config.max_memory_mb,
638
+ "max_lifetime" => config.max_worker_lifetime
639
+ }.compact
640
+ end
641
+
592
642
  # Runs once per heartbeat interval (not per message), so it's the right
593
643
  # place to emit connection-pool observability without touching any per-job
594
644
  # hot path. Reading the pool must never crash the beat — pool_stats already
@@ -697,7 +697,7 @@ module Pgbus
697
697
  def start_heartbeat
698
698
  @heartbeat = Heartbeat.new(
699
699
  kind: "dispatcher",
700
- metadata: { pid: ::Process.pid },
700
+ metadata: { pid: ::Process.pid, started_at: Time.now.to_f, supervisor_pid: ::Process.ppid },
701
701
  loop_tick_supplier: -> { @loop_tick_at.get }
702
702
  )
703
703
  @heartbeat.start
@@ -195,9 +195,11 @@ module Pgbus
195
195
  name = worker_config[:name] || worker_config["name"] || "anonymous"
196
196
  queues = worker_config[:queues] || worker_config["queues"] || [config.default_queue]
197
197
  threads = worker_config[:threads] || worker_config["threads"] || 5
198
+ processes = config.processes_for(worker_config)
198
199
  mode = banner_field { config.execution_mode_for(worker_config) }
199
200
  Pgbus.logger.info do
200
- "[Pgbus] boot: capsule=#{name} queues=#{Array(queues).join(",")} threads=#{threads} mode=#{mode}"
201
+ "[Pgbus] boot: capsule=#{name} queues=#{Array(queues).join(",")} threads=#{threads} " \
202
+ "processes=#{processes} mode=#{mode}"
201
203
  end
202
204
  end
203
205
  end
@@ -284,12 +286,7 @@ module Pgbus
284
286
  # --dispatcher-only CLI flags). Each role is gated by
285
287
  # config.role_enabled?, which returns true unless +config.roles+ has
286
288
  # been narrowed.
287
- if config.role_enabled?(:workers)
288
- # slot is the child's position in the config array — it keys the
289
- # crash-streak tracking so identically-configured siblings don't
290
- # share (and reset) each other's restart backoff.
291
- Array(config.workers).each_with_index { |worker_config, slot| fork_worker(worker_config, slot: slot) }
292
- end
289
+ boot_workers if config.role_enabled?(:workers)
293
290
 
294
291
  fork_dispatcher if config.role_enabled?(:dispatcher)
295
292
  boot_scheduler if config.role_enabled?(:scheduler)
@@ -297,7 +294,22 @@ module Pgbus
297
294
  boot_outbox_poller if config.role_enabled?(:outbox)
298
295
  end
299
296
 
300
- def fork_worker(worker_config, slot: nil)
297
+ # A capsule with processes: N forks N identical workers (issue #503).
298
+ # slot numbers every worker fork across all capsules — it keys the
299
+ # crash-streak tracking so identically-configured siblings (in one
300
+ # capsule or in several) don't share or reset each other's restart
301
+ # backoff. process_index is the fork's position within its capsule.
302
+ def boot_workers
303
+ slot = 0
304
+ Array(config.workers).each do |worker_config|
305
+ config.processes_for(worker_config).times do |process_index|
306
+ fork_worker(worker_config, slot: slot, process_index: process_index)
307
+ slot += 1
308
+ end
309
+ end
310
+ end
311
+
312
+ def fork_worker(worker_config, slot: nil, process_index: 0)
301
313
  note_intended_child
302
314
  queues = worker_config[:queues] || [config.default_queue]
303
315
  threads = worker_config[:threads] || 5
@@ -331,6 +343,8 @@ module Pgbus
331
343
  single_active_consumer: single_active, consumer_priority: priority,
332
344
  execution_mode: exec_mode, group_mode: grp_mode,
333
345
  read_ahead: config.read_ahead_for(worker_config),
346
+ recycle_limits: worker_config.slice(*Configuration::RECYCLE_LIMIT_KEYS),
347
+ capsule: worker_config[:name], process: process_label(worker_config, process_index),
334
348
  liveness_pipe: liveness_writer, wake_pipe: wake_reader
335
349
  )
336
350
  worker.run
@@ -352,7 +366,7 @@ module Pgbus
352
366
  close_pipe(wake_reader)
353
367
  register_fork_with_hub(pid, wake_writer, queues)
354
368
  @forks[pid] = {
355
- type: :worker, config: worker_config, slot: slot, spawned_at: monotonic_now,
369
+ type: :worker, config: worker_config, slot: slot, process_index: process_index, spawned_at: monotonic_now,
356
370
  liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false,
357
371
  wake_writer: wake_writer
358
372
  }
@@ -365,6 +379,13 @@ module Pgbus
365
379
  ErrorReporter.report(e, { action: "fork_worker", queues: queues })
366
380
  end
367
381
 
382
+ # "2/4" for the second of a capsule's four processes; nil for a
383
+ # single-process capsule, where it would only add noise.
384
+ def process_label(worker_config, process_index)
385
+ total = config.processes_for(worker_config)
386
+ "#{process_index + 1}/#{total}" if total > 1
387
+ end
388
+
368
389
  # Hand the hub a worker fork's routing entry: explicit queues as
369
390
  # physical names, "*" as the unconditional wildcard flag (the hub wakes
370
391
  # wildcard forks for every channel, so the fork's own resolved set never
@@ -682,7 +703,7 @@ module Pgbus
682
703
  def restart_child(info)
683
704
  case info[:type]
684
705
  when :worker
685
- fork_worker(info[:config], slot: info[:slot])
706
+ fork_worker(info[:config], slot: info[:slot], process_index: info[:process_index] || 0)
686
707
  when :dispatcher
687
708
  fork_dispatcher
688
709
  when :scheduler
@@ -882,7 +903,7 @@ module Pgbus
882
903
  def start_heartbeat
883
904
  @heartbeat = Heartbeat.new(
884
905
  kind: "supervisor",
885
- metadata: { pid: ::Process.pid, hostname: Socket.gethostname }
906
+ metadata: { pid: ::Process.pid, hostname: Socket.gethostname, started_at: Time.now.to_f }
886
907
  )
887
908
  @heartbeat.start
888
909
  end