pgbus 0.14.1 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +35 -0
  3. data/README.md +33 -0
  4. data/Rakefile +6 -1
  5. data/app/helpers/pgbus/application_helper.rb +7 -0
  6. data/app/models/pgbus/batch_entry.rb +78 -6
  7. data/app/models/pgbus/batch_execution.rb +28 -0
  8. data/app/models/pgbus/blocked_execution.rb +2 -2
  9. data/app/models/pgbus/uniqueness_key.rb +50 -4
  10. data/app/views/pgbus/batches/_batches_table.html.erb +3 -3
  11. data/app/views/pgbus/batches/show.html.erb +7 -7
  12. data/app/views/pgbus/dead_letter/show.html.erb +17 -0
  13. data/app/views/pgbus/events/_pending_table.html.erb +21 -0
  14. data/app/views/pgbus/jobs/show.html.erb +17 -0
  15. data/config/locales/da.yml +5 -2
  16. data/config/locales/de.yml +5 -2
  17. data/config/locales/en.yml +5 -2
  18. data/config/locales/es.yml +5 -2
  19. data/config/locales/fi.yml +5 -2
  20. data/config/locales/fr.yml +5 -2
  21. data/config/locales/it.yml +5 -2
  22. data/config/locales/ja.yml +5 -2
  23. data/config/locales/nb.yml +5 -2
  24. data/config/locales/nl.yml +5 -2
  25. data/config/locales/pt.yml +5 -2
  26. data/config/locales/sv.yml +5 -2
  27. data/lib/generators/pgbus/add_batch_callback_jobs_generator.rb +47 -0
  28. data/lib/generators/pgbus/add_batch_executions_generator.rb +44 -0
  29. data/lib/generators/pgbus/templates/add_batch_callback_jobs.rb.erb +13 -0
  30. data/lib/generators/pgbus/templates/add_batch_executions.rb.erb +50 -0
  31. data/lib/generators/pgbus/templates/initializer.rb.erb +2 -0
  32. data/lib/generators/pgbus/templates/migration.rb.erb +23 -2
  33. data/lib/pgbus/active_job/adapter.rb +146 -28
  34. data/lib/pgbus/active_job/batch_id.rb +48 -0
  35. data/lib/pgbus/active_job/current_attributes.rb +52 -0
  36. data/lib/pgbus/active_job/executor.rb +45 -9
  37. data/lib/pgbus/batch/sweep.rb +163 -0
  38. data/lib/pgbus/batch.rb +448 -63
  39. data/lib/pgbus/client/fair_read.rb +187 -0
  40. data/lib/pgbus/client.rb +221 -18
  41. data/lib/pgbus/concurrency/blocked_execution.rb +14 -1
  42. data/lib/pgbus/configuration.rb +81 -1
  43. data/lib/pgbus/current_attributes.rb +156 -0
  44. data/lib/pgbus/engine.rb +2 -0
  45. data/lib/pgbus/event.rb +7 -2
  46. data/lib/pgbus/event_bus/handler.rb +12 -2
  47. data/lib/pgbus/event_bus/publisher.rb +37 -2
  48. data/lib/pgbus/event_bus/subscriber.rb +4 -0
  49. data/lib/pgbus/fair_share.rb +110 -0
  50. data/lib/pgbus/generators/migration_detector.rb +30 -0
  51. data/lib/pgbus/instrumentation.rb +5 -0
  52. data/lib/pgbus/outbox/poller.rb +6 -7
  53. data/lib/pgbus/outbox.rb +8 -1
  54. data/lib/pgbus/process/consumer.rb +38 -1
  55. data/lib/pgbus/process/dispatcher.rb +41 -17
  56. data/lib/pgbus/process/worker.rb +41 -0
  57. data/lib/pgbus/recurring/schedule.rb +12 -1
  58. data/lib/pgbus/testing.rb +2 -1
  59. data/lib/pgbus/uniqueness.rb +35 -7
  60. data/lib/pgbus/version.rb +1 -1
  61. data/lib/pgbus/web/data_source.rb +47 -9
  62. data/lib/pgbus/web/job_context.rb +80 -0
  63. data/lib/pgbus.rb +2 -0
  64. metadata +13 -1
@@ -51,6 +51,7 @@ module Pgbus
51
51
  # Enumerated so set_maintenance_timestamp can validate its argument.
52
52
  MAINTENANCE_TIMESTAMPS = %i[
53
53
  @last_cleanup_at @last_reap_at @last_concurrency_at @last_batch_cleanup_at
54
+ @last_batch_sweep_at
54
55
  @last_recurring_cleanup_at @last_archive_compaction_at @last_stream_archive_compaction_at
55
56
  @last_outbox_cleanup_at @last_job_lock_cleanup_at @last_stats_cleanup_at
56
57
  @last_orphan_stream_sweep_at @last_table_maintenance_at
@@ -107,7 +108,7 @@ module Pgbus
107
108
 
108
109
  # Test seam: set one of the @last_*_at maintenance timestamps so a task
109
110
  # becomes (or stops being) due on the next run_maintenance. The timestamps
110
- # are per-task monotonic clocks with no constructor injection point (all 12
111
+ # are per-task monotonic clocks with no constructor injection point (all
111
112
  # default to monotonic_now at construction), so a post-construction setter
112
113
  # is the minimal way to drive the due/not-due logic deterministically.
113
114
  # ivar must be one of the recognized @last_*_at names.
@@ -233,6 +234,7 @@ module Pgbus
233
234
  results << run_if_due(now, :@last_reap_at, REAP_INTERVAL) { reap_stale_processes }
234
235
  results << run_if_due(now, :@last_concurrency_at, CONCURRENCY_INTERVAL) { cleanup_concurrency }
235
236
  results << run_if_due(now, :@last_batch_cleanup_at, BATCH_CLEANUP_INTERVAL) { cleanup_batches }
237
+ results << run_if_due(now, :@last_batch_sweep_at, config.batch_sweep_interval) { sweep_stalled_batches }
236
238
  results << run_if_due(now, :@last_recurring_cleanup_at, RECURRING_CLEANUP_INTERVAL) { cleanup_recurring_executions }
237
239
  results << run_if_due(now, :@last_archive_compaction_at, ARCHIVE_COMPACTION_INTERVAL) { compact_archives }
238
240
  results << run_if_due(now, :@last_stream_archive_compaction_at, ARCHIVE_COMPACTION_INTERVAL) { prune_stream_archives }
@@ -373,13 +375,23 @@ module Pgbus
373
375
  end
374
376
 
375
377
  def cleanup_batches
376
- deleted = Batch.cleanup(older_than: Time.current - (7 * 24 * 3600)) # 7 days
378
+ retention = config.batch_retention
379
+ return unless retention&.positive?
380
+
381
+ deleted = Batch.cleanup(older_than: Time.current - retention)
377
382
  Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} finished batches" } if deleted.positive?
378
383
  rescue StandardError => e
379
384
  log_maintenance_failure("Batch cleanup", e)
380
385
  e
381
386
  end
382
387
 
388
+ def sweep_stalled_batches
389
+ Batch.sweep_stalled
390
+ rescue StandardError => e
391
+ log_maintenance_failure("Batch sweep", e)
392
+ e
393
+ end
394
+
383
395
  def cleanup_stats
384
396
  retention = config.stats_retention
385
397
  return unless retention&.positive?
@@ -446,34 +458,46 @@ module Pgbus
446
458
  # younger could be a freshly-acquired lock whose send_message hasn't
447
459
  # committed yet.
448
460
  threshold = Time.current - (config.visibility_timeout * 2)
461
+ candidates = keys.select { |key| key.created_at && key.created_at < threshold && key.queue_name }
462
+ return 0 if candidates.empty?
449
463
 
450
- orphaned = keys.select do |key|
451
- next false unless key.created_at && key.created_at < threshold
452
- next false unless key.queue_name
453
-
454
- message_gone?(key)
455
- end
464
+ bound, unbound = candidates.partition { |key| bound_lock?(key) }
465
+ orphaned = bound.select { |key| message_gone?(key) }
466
+ orphaned.concat(gone_unbound_locks(unbound))
456
467
 
457
468
  return 0 if orphaned.empty?
458
469
 
459
470
  UniquenessKey.where(lock_key: orphaned.map(&:lock_key)).delete_all
460
471
  end
461
472
 
462
- # Returns true if the message referenced by this lock is definitely gone
463
- # from the queue. Returns false otherwise (message present, or unknown).
473
+ def bound_lock?(key)
474
+ !Uniqueness.placeholder?(queue_name: key.queue_name, msg_id: key.msg_id)
475
+ end
476
+
477
+ # Unbound rows (synthetic "pending" queue and/or msg_id=0) must not probe
478
+ # pgmq.q_<prefix>_pending — that table does not exist, message_exists?
479
+ # returns nil, and the reaper would keep the lock forever (issue #418).
480
+ # Scan every live queue for the payload key instead.
481
+ def gone_unbound_locks(unbound)
482
+ return [] if unbound.empty?
483
+
484
+ present = Pgbus.client.uniqueness_keys_present(unbound.map(&:lock_key))
485
+ unbound.reject { |key| present.include?(key.lock_key) }
486
+ end
487
+
488
+ # Returns true if the message referenced by this bound lock is definitely
489
+ # gone from the queue. Returns false otherwise (message present, or unknown).
464
490
  #
465
491
  # Routes through Pgbus::Client#message_exists? so all PGMQ access stays
466
492
  # behind the client interface. The client returns nil when it can't
467
493
  # determine the answer (queue table missing, etc.); we treat that as
468
494
  # "still here" — the reaper must NEVER delete a lock when in doubt.
469
495
  def message_gone?(key)
470
- msg_id = key.msg_id.to_i
471
- result = if msg_id.positive?
472
- Pgbus.client.message_exists?(key.queue_name, msg_id: msg_id)
473
- else
474
- Pgbus.client.message_exists?(key.queue_name, uniqueness_key: key.lock_key)
475
- end
476
-
496
+ result = Pgbus.client.message_exists?(
497
+ key.queue_name,
498
+ msg_id: key.msg_id.to_i,
499
+ uniqueness_key: key.lock_key
500
+ )
477
501
  result == false
478
502
  rescue StandardError => e
479
503
  Pgbus.logger.warn { "[Pgbus] Reap check failed for #{key.lock_key}: #{e.message}" }
@@ -54,6 +54,11 @@ module Pgbus
54
54
  raise ArgumentError,
55
55
  "Invalid group_mode: #{@group_mode.inspect}. Must be nil, :fifo, or :round_robin"
56
56
  end
57
+ if FairShare.enabled?(config) && (@group_mode || config.group_mode)
58
+ raise Pgbus::ConfigurationError,
59
+ "fair_share and group_mode are mutually exclusive — this worker has group_mode " \
60
+ "#{(@group_mode || config.group_mode).inspect} while config.fair_share is set"
61
+ end
57
62
  @single_active_consumer = single_active_consumer
58
63
  @consumer_priority = consumer_priority
59
64
  @lifecycle = Lifecycle.new
@@ -165,6 +170,7 @@ module Pgbus
165
170
  setup_signals
166
171
  start_heartbeat
167
172
  resolve_wildcard_queues
173
+ ensure_fair_indexes
168
174
  start_wake_source
169
175
  @lifecycle.transition_to!(:running)
170
176
  Pgbus.logger.info do
@@ -284,6 +290,8 @@ module Pgbus
284
290
  fetch_prioritized(active_queues, qty)
285
291
  elsif @group_mode
286
292
  fetch_grouped(active_queues, qty)
293
+ elsif fair_share_enabled?
294
+ fetch_fair(active_queues, qty)
287
295
  elsif active_queues.size == 1
288
296
  queue = active_queues.first
289
297
  messages = Pgbus.client.read_batch(queue, qty: qty) || []
@@ -391,6 +399,38 @@ module Pgbus
391
399
  results
392
400
  end
393
401
 
402
+ # Fair share (issue #426): one weighted-interleave read per queue, in
403
+ # list order with the remaining capacity — strict priority ACROSS queues
404
+ # (the capsule DSL contract), fair WITHIN each queue. Same loop shape as
405
+ # fetch_prioritized; under priority_levels the client composes both.
406
+ def fetch_fair(active_queues, qty)
407
+ remaining = qty
408
+ results = []
409
+
410
+ active_queues.each do |queue|
411
+ break if remaining <= 0
412
+
413
+ messages = Pgbus.client.read_batch_fair(queue, qty: remaining) || []
414
+ messages.each { |m| results << [queue, m] }
415
+ remaining -= messages.size
416
+ end
417
+
418
+ results
419
+ end
420
+
421
+ def fair_share_enabled?
422
+ FairShare.enabled?(config)
423
+ end
424
+
425
+ # Build the fair index on every queue this worker serves (idempotent,
426
+ # memoized in the client, CONCURRENTLY so a populated queue is never
427
+ # write-locked). Queues created later get theirs at creation time.
428
+ def ensure_fair_indexes
429
+ return unless fair_share_enabled?
430
+
431
+ queues.each { |q| Pgbus.client.ensure_fair_index(q) }
432
+ end
433
+
394
434
  def priority_enabled?
395
435
  config.priority_levels && config.priority_levels > 1
396
436
  end
@@ -448,6 +488,7 @@ module Pgbus
448
488
  return if @last_wildcard_resolve && (monotonic_now - @last_wildcard_resolve) < WILDCARD_REFRESH_INTERVAL
449
489
 
450
490
  resolve_wildcard_queues
491
+ ensure_fair_indexes
451
492
  end
452
493
 
453
494
  # When a "relation does not exist" error occurs, the queue was deleted.
@@ -29,7 +29,8 @@ module Pgbus
29
29
  payload = inject_uniqueness_metadata(task, payload)
30
30
 
31
31
  Pgbus.client.ensure_queue(queue)
32
- Pgbus.client.send_message(queue, payload, headers: headers)
32
+ msg_id = Pgbus.client.send_message(queue, payload, headers: headers)
33
+ bind_uniqueness_lock(acquired_key, queue, msg_id)
33
34
 
34
35
  Pgbus.logger.info do
35
36
  "[Pgbus] Enqueued recurring task #{task.key} (#{task.class_name || task.command}) " \
@@ -159,6 +160,16 @@ module Pgbus
159
160
  :already_locked # Fail closed — skip enqueue when lock check errors
160
161
  end
161
162
 
163
+ # Point the pre-produce lock at the real PGMQ msg_id. Fail-soft: a bind
164
+ # error must not roll back an already-produced message (issue #418).
165
+ def bind_uniqueness_lock(key, queue, msg_id)
166
+ return unless key.is_a?(String)
167
+
168
+ Uniqueness.bind_lock(key, queue_name: queue, msg_id: msg_id)
169
+ rescue StandardError => e
170
+ Pgbus.logger.warn { "[Pgbus] Uniqueness bind failed: #{e.message}" }
171
+ end
172
+
162
173
  # Release a uniqueness lock. Safe to call with nil or :already_locked.
163
174
  def release_uniqueness_lock(key)
164
175
  return if key.nil? || key == :already_locked
data/lib/pgbus/testing.rb CHANGED
@@ -54,7 +54,8 @@ module Pgbus
54
54
  break unless event
55
55
 
56
56
  Pgbus::EventBus::Registry.instance.handlers_for(event.routing_key).each do |subscriber|
57
- subscriber.handler_class.new.handle(event)
57
+ # Restore the publisher's Current (issue #431) like the consumer does.
58
+ Pgbus::CurrentAttributes.restore(event.context) { subscriber.handler_class.new.handle(event) }
58
59
  end
59
60
 
60
61
  @mutex.synchronize { @events.shift }
@@ -10,9 +10,10 @@ module Pgbus
10
10
  # at any time — from enqueue through completion.
11
11
  #
12
12
  # Lock lifecycle (advisory lock + thin lookup table):
13
- # 1. Enqueue: pg_advisory_xact_lock serializes concurrent attempts,
14
- # then INSERT INTO pgbus_uniqueness_keys ON CONFLICT DO NOTHING.
15
- # The lock row lives as long as the job is in the queue or executing.
13
+ # 1. Enqueue: INSERT INTO pgbus_uniqueness_keys ON CONFLICT DO NOTHING
14
+ # (logical queue, msg_id=0). After send_message, bind! writes the
15
+ # real msg_id. The lock row lives as long as the job is in the queue
16
+ # or executing.
16
17
  # 2. Execution: PGMQ's visibility timeout is the execution lock —
17
18
  # no separate claim_for_execution step needed.
18
19
  # 3. Completion/DLQ: DELETE FROM pgbus_uniqueness_keys WHERE lock_key = ?.
@@ -43,6 +44,10 @@ module Pgbus
43
44
 
44
45
  METADATA_KEY = "pgbus_uniqueness_key"
45
46
  STRATEGY_KEY = "pgbus_uniqueness_strategy"
47
+ # Synthetic queue stored before produce when the caller does not know the
48
+ # real queue yet. Never a live PGMQ queue — the reaper must not probe
49
+ # pgmq.q_<prefix>_pending for these rows (issue #418).
50
+ PLACEHOLDER_QUEUE = "pending"
46
51
 
47
52
  VALID_STRATEGIES = %i[until_executed while_executing].freeze
48
53
  VALID_CONFLICTS = %i[reject discard log].freeze
@@ -174,19 +179,25 @@ module Pgbus
174
179
  UniquenessKey.acquire!(key, queue_name: queue_name, msg_id: msg_id)
175
180
  else
176
181
  # Pre-produce check: use advisory lock + ON CONFLICT
177
- UniquenessKey.acquire!(key, queue_name: queue_name || "pending", msg_id: msg_id || 0)
182
+ UniquenessKey.acquire!(key, queue_name: queue_name || PLACEHOLDER_QUEUE, msg_id: msg_id || 0)
178
183
  end
179
184
  acquired ? :acquired : :locked
180
185
  end
181
186
 
182
187
  # Acquire the uniqueness lock at execution time (:while_executing only).
183
188
  # Returns true if acquired, false if another instance is running.
184
- def acquire_execution_lock(key, payload)
189
+ #
190
+ # Bound to the message being executed so a row left behind by a crashed
191
+ # attempt of the SAME message is re-acquired on retry instead of locking
192
+ # that message out until it dead-letters (issue #423). Callers without a
193
+ # message (legacy signature) fall back to the unbound placeholder.
194
+ def acquire_execution_lock(key, payload, msg_id: 0, queue_name: nil)
185
195
  strategy = extract_strategy(payload)
186
196
  return true unless strategy == :while_executing
187
197
 
188
- queue_name = payload["queue_name"] || "unknown"
189
- UniquenessKey.acquire!(key, queue_name: queue_name, msg_id: 0)
198
+ queue_name ||= payload["queue_name"] || "unknown"
199
+ UniquenessKey.acquire!(key, queue_name: queue_name, msg_id: msg_id.to_i,
200
+ reacquire_same_message: msg_id.to_i.positive?)
190
201
  end
191
202
 
192
203
  # Release the uniqueness lock after execution completes.
@@ -195,6 +206,23 @@ module Pgbus
195
206
 
196
207
  UniquenessKey.release!(key)
197
208
  end
209
+
210
+ # Point a pre-produce lock at the real queue + PGMQ msg_id after send.
211
+ # No-op when key is nil or msg_id is not a positive integer (the reaper
212
+ # treats those rows as unbound and scans live queues instead).
213
+ def bind_lock(key, queue_name:, msg_id:)
214
+ return unless key
215
+ return unless msg_id.to_i.positive?
216
+
217
+ UniquenessKey.bind!(key, queue_name: queue_name, msg_id: msg_id)
218
+ end
219
+
220
+ # True when the uniqueness row is not yet bound to a real PGMQ message.
221
+ # Covers the synthetic pending queue and any msg_id=0 placeholder
222
+ # (recurring scheduler, bind-not-yet-run, bind failure).
223
+ def placeholder?(queue_name:, msg_id:)
224
+ msg_id.to_i <= 0 || queue_name.to_s == PLACEHOLDER_QUEUE
225
+ end
198
226
  end
199
227
  end
200
228
  end
data/lib/pgbus/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Pgbus
4
- VERSION = "0.14.1"
4
+ VERSION = "0.15.0"
5
5
  end
@@ -669,7 +669,9 @@ module Pgbus
669
669
 
670
670
  # Batches
671
671
  def batches(limit: 100)
672
- BatchEntry.order(created_at: :desc).limit(limit).map { |r| format_batch(r) }
672
+ records = BatchEntry.order(created_at: :desc).limit(limit).to_a
673
+ pending = pending_jobs_by_batch(records.map(&:batch_id))
674
+ records.map { |r| format_batch(r, pending_jobs: pending[r.batch_id]) }
673
675
  rescue StandardError => e
674
676
  Pgbus.logger.debug { "[Pgbus::Web] Error fetching batches: #{e.message}" }
675
677
  []
@@ -679,11 +681,14 @@ module Pgbus
679
681
  record = BatchEntry.find_by(batch_id: batch_id)
680
682
  return nil unless record
681
683
 
682
- format_batch(record).merge(
684
+ pending = pending_jobs_by_batch([batch_id])
685
+ failure_callback = callback_display_name(record, :on_failure) || failure_callback_class(record)
686
+ format_batch(record, pending_jobs: pending[batch_id]).merge(
683
687
  properties: record.properties,
684
- on_finish_class: record.on_finish_class,
685
- on_success_class: record.on_success_class,
686
- on_discard_class: record.on_discard_class
688
+ on_finish_class: callback_display_name(record, :on_finish) || record.on_finish_class,
689
+ on_success_class: callback_display_name(record, :on_success) || record.on_success_class,
690
+ on_failure_class: failure_callback,
691
+ on_discard_class: failure_callback
687
692
  )
688
693
  rescue StandardError => e
689
694
  Pgbus.logger.debug { "[Pgbus::Web] Error fetching batch #{batch_id}: #{e.message}" }
@@ -1351,10 +1356,12 @@ module Pgbus
1351
1356
  end
1352
1357
  end
1353
1358
 
1354
- def format_batch(record)
1359
+ def format_batch(record, pending_jobs: nil)
1355
1360
  total = record.total_jobs
1356
- done = record.completed_jobs + record.discarded_jobs
1361
+ failed = failure_job_count(record)
1362
+ done = record.completed_jobs + failed
1357
1363
  pct = total.positive? ? ((done * 100) / total) : 100
1364
+ pending = pending_jobs || [total - done, 0].max
1358
1365
 
1359
1366
  {
1360
1367
  batch_id: record.batch_id,
@@ -1362,14 +1369,45 @@ module Pgbus
1362
1369
  status: record.status,
1363
1370
  total_jobs: total,
1364
1371
  completed_jobs: record.completed_jobs,
1365
- discarded_jobs: record.discarded_jobs,
1366
- failed_jobs: record.failed_jobs,
1372
+ failed_jobs: failed,
1373
+ discarded_jobs: failed,
1374
+ pending_jobs: pending,
1367
1375
  progress_pct: pct,
1368
1376
  created_at: record.created_at,
1369
1377
  finished_at: record.finished_at
1370
1378
  }
1371
1379
  end
1372
1380
 
1381
+ def failure_job_count(record)
1382
+ record.respond_to?(:discarded_jobs) ? record.discarded_jobs.to_i : record.failed_jobs.to_i
1383
+ end
1384
+
1385
+ # A callback configured as an ActiveJob instance lives in the *_job jsonb
1386
+ # column, not the legacy *_class string — show its job_class either way.
1387
+ def callback_display_name(record, slot)
1388
+ column = "#{slot}_job"
1389
+ return nil unless record.respond_to?(column)
1390
+
1391
+ record.public_send(column).presence&.dig("job_class")
1392
+ end
1393
+
1394
+ def failure_callback_class(record)
1395
+ if record.respond_to?(:on_failure_class) && record.on_failure_class.present?
1396
+ record.on_failure_class
1397
+ elsif record.respond_to?(:on_discard_class)
1398
+ record.on_discard_class
1399
+ end
1400
+ end
1401
+
1402
+ def pending_jobs_by_batch(batch_ids)
1403
+ return {} unless Batch.executions_migrated?
1404
+ return {} if batch_ids.empty?
1405
+
1406
+ BatchExecution.where(batch_id: batch_ids).group(:batch_id).count
1407
+ rescue StandardError
1408
+ {}
1409
+ end
1410
+
1373
1411
  def sanitize_name(name)
1374
1412
  QueueNameValidator.sanitize!(name)
1375
1413
  end
@@ -0,0 +1,80 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module Pgbus
6
+ module Web
7
+ # Presents the Current attributes persisted in a job payload
8
+ # (Pgbus::CurrentAttributes::METADATA_KEY) for the dashboard's Context
9
+ # card: { "Current" => { "tenant" => "gid://app/Tenant/42", ... } }.
10
+ #
11
+ # Pure presentation: ActiveJob argument encodings are unwrapped to display
12
+ # strings (a GlobalID shows its gid, a serialized Symbol/Time its value) —
13
+ # nothing is constantized or located. Callers pass the payload through
14
+ # PayloadFilter first (ApplicationHelper#pgbus_parse_message does), so
15
+ # sensitive values arrive already redacted.
16
+ module JobContext
17
+ AJ_GLOBALID = "_aj_globalid"
18
+ AJ_SERIALIZED = "_aj_serialized"
19
+ AJ_META_PREFIX = "_aj_"
20
+
21
+ module_function
22
+
23
+ def from_payload(payload)
24
+ data = parse(payload)
25
+ stored = data && data[Pgbus::CurrentAttributes::METADATA_KEY]
26
+ return nil unless stored.is_a?(Hash) && stored.any?
27
+
28
+ stored.to_h do |klass_name, attrs|
29
+ [klass_name.to_s, present_attrs(attrs)]
30
+ end
31
+ end
32
+
33
+ def present_attrs(attrs)
34
+ return {} unless attrs.is_a?(Hash)
35
+
36
+ attrs.each_with_object({}) do |(name, value), out|
37
+ next if name.to_s.start_with?(AJ_META_PREFIX)
38
+
39
+ out[name.to_s] = display(value)
40
+ end
41
+ end
42
+
43
+ # String form of a value for the card. Hashes/arrays that are ActiveJob
44
+ # encodings collapse to their payload; anything else renders as JSON
45
+ # (stable across Ruby versions — Hash#inspect changed in 3.4).
46
+ def display(value)
47
+ unwrapped = unwrap(value)
48
+ unwrapped.is_a?(String) ? unwrapped : JSON.generate(unwrapped)
49
+ end
50
+
51
+ def unwrap(value)
52
+ case value
53
+ when Hash
54
+ return value[AJ_GLOBALID] if value.key?(AJ_GLOBALID)
55
+ return unwrap(value["value"]) if value.key?(AJ_SERIALIZED) && value.key?("value")
56
+
57
+ value.each_with_object({}) do |(k, v), out|
58
+ next if k.to_s.start_with?(AJ_META_PREFIX)
59
+
60
+ out[k] = unwrap(v)
61
+ end
62
+ when Array
63
+ value.map { |v| unwrap(v) }
64
+ else
65
+ value
66
+ end
67
+ end
68
+
69
+ def parse(payload)
70
+ case payload
71
+ when Hash then payload
72
+ when String then JSON.parse(payload)
73
+ end
74
+ rescue JSON::ParserError
75
+ nil
76
+ end
77
+ private_class_method :parse
78
+ end
79
+ end
80
+ end
data/lib/pgbus.rb CHANGED
@@ -49,6 +49,8 @@ module Pgbus
49
49
  # comes back with a msg_id count that doesn't match the number of jobs sent —
50
50
  # a data-integrity signal that some jobs may not have been persisted.
51
51
  class EnqueueError < Error; end
52
+ # A Current attribute could not be serialized for job persistence (issue #430).
53
+ class CurrentAttributesError < Error; end
52
54
 
53
55
  # Raised by the execution pools when work can't be accepted: the pool is
54
56
  # shutting down, or it's momentarily at capacity. Consumer-reachable during
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pgbus
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.14.1
4
+ version: 0.15.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Mikael Henriksson
@@ -150,6 +150,7 @@ files:
150
150
  - app/helpers/pgbus/streams_helper.rb
151
151
  - app/models/pgbus/application_record.rb
152
152
  - app/models/pgbus/batch_entry.rb
153
+ - app/models/pgbus/batch_execution.rb
153
154
  - app/models/pgbus/blocked_execution.rb
154
155
  - app/models/pgbus/job_stat.rb
155
156
  - app/models/pgbus/outbox_entry.rb
@@ -210,6 +211,8 @@ files:
210
211
  - exe/pgbus
211
212
  - exe/pgbus-health
212
213
  - lib/active_job/queue_adapters/pgbus_adapter.rb
214
+ - lib/generators/pgbus/add_batch_callback_jobs_generator.rb
215
+ - lib/generators/pgbus/add_batch_executions_generator.rb
213
216
  - lib/generators/pgbus/add_failed_events_index_generator.rb
214
217
  - lib/generators/pgbus/add_job_stats_generator.rb
215
218
  - lib/generators/pgbus/add_job_stats_latency_generator.rb
@@ -225,6 +228,8 @@ files:
225
228
  - lib/generators/pgbus/install_generator.rb
226
229
  - lib/generators/pgbus/migrate_job_locks_generator.rb
227
230
  - lib/generators/pgbus/migration_path.rb
231
+ - lib/generators/pgbus/templates/add_batch_callback_jobs.rb.erb
232
+ - lib/generators/pgbus/templates/add_batch_executions.rb.erb
228
233
  - lib/generators/pgbus/templates/add_failed_events_unique_index.rb.erb
229
234
  - lib/generators/pgbus/templates/add_job_stats.rb.erb
230
235
  - lib/generators/pgbus/templates/add_job_stats_latency.rb.erb
@@ -251,9 +256,12 @@ files:
251
256
  - lib/generators/pgbus/upgrade_pgmq_generator.rb
252
257
  - lib/pgbus.rb
253
258
  - lib/pgbus/active_job/adapter.rb
259
+ - lib/pgbus/active_job/batch_id.rb
260
+ - lib/pgbus/active_job/current_attributes.rb
254
261
  - lib/pgbus/active_job/executor.rb
255
262
  - lib/pgbus/autovacuum_tuning.rb
256
263
  - lib/pgbus/batch.rb
264
+ - lib/pgbus/batch/sweep.rb
257
265
  - lib/pgbus/bus_record.rb
258
266
  - lib/pgbus/circuit_breaker.rb
259
267
  - lib/pgbus/cli.rb
@@ -262,6 +270,7 @@ files:
262
270
  - lib/pgbus/client.rb
263
271
  - lib/pgbus/client/connection_health.rb
264
272
  - lib/pgbus/client/ensure_stream_queue.rb
273
+ - lib/pgbus/client/fair_read.rb
265
274
  - lib/pgbus/client/notify_stream.rb
266
275
  - lib/pgbus/client/read_after.rb
267
276
  - lib/pgbus/client/resizable_pool.rb
@@ -270,6 +279,7 @@ files:
270
279
  - lib/pgbus/concurrency/semaphore.rb
271
280
  - lib/pgbus/configuration.rb
272
281
  - lib/pgbus/configuration/capsule_dsl.rb
282
+ - lib/pgbus/current_attributes.rb
273
283
  - lib/pgbus/database_tasks_guard.rb
274
284
  - lib/pgbus/dedicated_connection.rb
275
285
  - lib/pgbus/dedup_cache.rb
@@ -285,6 +295,7 @@ files:
285
295
  - lib/pgbus/execution_pools/async_pool.rb
286
296
  - lib/pgbus/execution_pools/thread_pool.rb
287
297
  - lib/pgbus/failed_event_recorder.rb
298
+ - lib/pgbus/fair_share.rb
288
299
  - lib/pgbus/generators/database_target_detector.rb
289
300
  - lib/pgbus/generators/migration_detector.rb
290
301
  - lib/pgbus/health_probe.rb
@@ -387,6 +398,7 @@ files:
387
398
  - lib/pgbus/web/data_source.rb
388
399
  - lib/pgbus/web/health_app.rb
389
400
  - lib/pgbus/web/health_server.rb
401
+ - lib/pgbus/web/job_context.rb
390
402
  - lib/pgbus/web/metrics_serializer.rb
391
403
  - lib/pgbus/web/payload_filter.rb
392
404
  - lib/pgbus/web/stream_app.rb