pgbus 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +59 -6
  3. data/README.md +3 -2
  4. data/app/frontend/pgbus/modules/charts.js +18 -0
  5. data/app/frontend/pgbus/modules/charts.min.js +1 -1
  6. data/app/frontend/pgbus/modules/charts.min.js.map +1 -1
  7. data/app/helpers/pgbus/workflow_insights_helper.rb +8 -0
  8. data/app/helpers/pgbus/workflows_helper.rb +2 -2
  9. data/app/models/pgbus/blocked_execution.rb +12 -0
  10. data/app/models/pgbus/uniqueness_key.rb +88 -3
  11. data/app/models/pgbus/workflow_entry.rb +3 -2
  12. data/app/models/pgbus/workflow_step.rb +7 -0
  13. data/app/views/pgbus/workflows/_timeline.html.erb +11 -1
  14. data/app/views/pgbus/workflows/definitions/_step_card.html.erb +3 -0
  15. data/config/locales/da.yml +7 -1
  16. data/config/locales/de.yml +7 -1
  17. data/config/locales/en.yml +7 -1
  18. data/config/locales/es.yml +7 -1
  19. data/config/locales/fi.yml +7 -1
  20. data/config/locales/fr.yml +7 -1
  21. data/config/locales/it.yml +7 -1
  22. data/config/locales/ja.yml +7 -1
  23. data/config/locales/nb.yml +7 -1
  24. data/config/locales/nl.yml +7 -1
  25. data/config/locales/pt.yml +7 -1
  26. data/config/locales/sv.yml +7 -1
  27. data/lib/generators/pgbus/templates/add_workflows.rb.erb +3 -1
  28. data/lib/generators/pgbus/templates/migration.rb.erb +3 -1
  29. data/lib/pgbus/active_job/adapter.rb +81 -3
  30. data/lib/pgbus/active_job/delivery.rb +20 -6
  31. data/lib/pgbus/active_job/executor.rb +18 -8
  32. data/lib/pgbus/concurrency/blocked_execution.rb +14 -6
  33. data/lib/pgbus/configuration/workflow_settings.rb +1 -1
  34. data/lib/pgbus/configuration.rb +43 -1
  35. data/lib/pgbus/dead_letter_header.rb +9 -1
  36. data/lib/pgbus/instrumentation.rb +3 -2
  37. data/lib/pgbus/integrations/appsignal/dashboard.json +30 -1
  38. data/lib/pgbus/mcp/tools/workflow_stats_tool.rb +1 -1
  39. data/lib/pgbus/mcp/tools/workflows_attention_tool.rb +1 -1
  40. data/lib/pgbus/metrics/workflow_metrics.rb +11 -1
  41. data/lib/pgbus/process/consumer.rb +2 -1
  42. data/lib/pgbus/process/dispatcher.rb +13 -3
  43. data/lib/pgbus/process/worker.rb +10 -1
  44. data/lib/pgbus/recurring/schedule.rb +1 -1
  45. data/lib/pgbus/streams/presence.rb +65 -36
  46. data/lib/pgbus/uniqueness.rb +23 -6
  47. data/lib/pgbus/version.rb +1 -1
  48. data/lib/pgbus/web/data_source/workflow_list.rb +52 -7
  49. data/lib/pgbus/web/data_source.rb +123 -15
  50. data/lib/pgbus/web/workflow_insights_report.rb +11 -3
  51. data/lib/pgbus/web/workflow_state.rb +19 -3
  52. data/lib/pgbus/web/workflow_timeline.rb +23 -6
  53. data/lib/pgbus/workflow/instance.rb +36 -3
  54. data/lib/pgbus/workflow/runner.rb +2 -2
  55. data/lib/pgbus/workflow/signal_peek.rb +47 -0
  56. data/lib/pgbus/workflow/signals.rb +31 -10
  57. data/lib/pgbus/workflow/step_definition.rb +49 -5
  58. data/lib/pgbus/workflow/step_execution.rb +27 -10
  59. data/lib/pgbus/workflow/wait_timeout.rb +56 -4
  60. data/lib/pgbus/workflow.rb +11 -5
  61. data/sig/pgbus/configuration.rbs +1 -0
  62. data/sig/pgbus.rbs +43 -0
  63. metadata +2 -1
@@ -33,7 +33,7 @@ module Pgbus
33
33
  FailedEventRecorder.clear!(queue_name: queue_name, msg_id: message.msg_id.to_i)
34
34
  signal_concurrency(payload)
35
35
  signal_batch_discarded(payload)
36
- Uniqueness.release_lock(Uniqueness.extract_key(payload))
36
+ release_uniqueness_lock(Uniqueness.extract_key(payload), queue_name, message)
37
37
  record_stat(payload, queue_name, "dead_lettered", execution_start, message: message)
38
38
  instrument(
39
39
  "pgbus.job_dead_lettered",
@@ -100,7 +100,10 @@ module Pgbus
100
100
  assign_batch_id(job, payload)
101
101
  Pgbus.logger.debug { "[Pgbus::Executor] running #{log_tag(message, queue_name)} job_class=#{job_class}" }
102
102
  with_visibility_heartbeat(job, queue_name, msg_id, source_queue, payload) do
103
- Delivery.with(msg_id: msg_id, read_ct: read_count, queue_name: queue_name) { execute_job(job) }
103
+ Delivery.with(msg_id: msg_id, read_ct: read_count, queue_name: queue_name,
104
+ enqueued_at: message.enqueued_at, job_id: payload["job_id"]) do
105
+ execute_job(job)
106
+ end
104
107
  end
105
108
  # retry_on re-enqueues from inside perform_now and returns normally:
106
109
  # this attempt is done (archive it) but the job is not — the retry
@@ -121,7 +124,7 @@ module Pgbus
121
124
  end
122
125
  Pgbus.logger.debug { "[Pgbus::Executor] archived #{log_tag(message, queue_name)} job_class=#{job_class}" }
123
126
  job_succeeded = true
124
- release_uniqueness_lock(uniqueness_key)
127
+ release_uniqueness_lock(uniqueness_key, queue_name, message)
125
128
  # Only FailedEventRecorder.record! writes this table, and only after a
126
129
  # failed attempt, so a first delivery has no row: skip the per-job
127
130
  # DELETE round trip (issue #484). A stale row from a dropped-and-
@@ -146,7 +149,7 @@ module Pgbus
146
149
  # A failed :while_executing attempt is no longer executing — release
147
150
  # so the retry (same message, after VT) can acquire. :until_executed
148
151
  # keeps its lock until success/DLQ by design (#126, #333).
149
- release_uniqueness_lock(uniqueness_key) if uniqueness_strategy == :while_executing
152
+ release_uniqueness_lock(uniqueness_key, queue_name, message) if uniqueness_strategy == :while_executing
150
153
  instrument(
151
154
  "pgbus.job_failed",
152
155
  queue: queue_name,
@@ -192,10 +195,15 @@ module Pgbus
192
195
  job.batch_id = batch_id
193
196
  end
194
197
 
195
- def release_uniqueness_lock(key)
198
+ # Releases only the row this message holds, never the key as such: a
199
+ # row the job moved onto its retry (issue #564), or a successor's, stays.
200
+ # The queue is spelled as the row was bound (Uniqueness.lock_queue_name),
201
+ # as for release_duplicate_execution_lock.
202
+ def release_uniqueness_lock(key, queue_name, message)
196
203
  return unless key
197
204
 
198
- Uniqueness.release_lock(key)
205
+ UniquenessKey.release_held_by!(key, queue_name: Uniqueness.lock_queue_name(queue_name, config: config),
206
+ msg_id: message.msg_id.to_i, enqueued_at: message.enqueued_at)
199
207
  rescue StandardError => e
200
208
  Pgbus.logger.warn { "[Pgbus] Uniqueness release failed: #{e.message}" }
201
209
  end
@@ -425,7 +433,8 @@ module Pgbus
425
433
 
426
434
  # Same queue the execution lock was acquired under, so the match
427
435
  # cannot land on another queue's message of the same id.
428
- UniquenessKey.release_if_bound!(uniqueness_key, queue_name: queue_name, msg_id: msg_id)
436
+ UniquenessKey.release_if_bound!(uniqueness_key, queue_name: Uniqueness.lock_queue_name(queue_name, config: config),
437
+ msg_id: msg_id)
429
438
  rescue StandardError => e
430
439
  Pgbus.logger.warn { "[Pgbus] Duplicate-execution lock release failed: #{e.message}" }
431
440
  end
@@ -442,7 +451,8 @@ module Pgbus
442
451
  end
443
452
  headers = DeadLetterHeader.build(
444
453
  existing: message.headers, reason: DeadLetterHeader::REASON_MAX_RETRIES, source: "worker",
445
- source_queue: source_queue || config.queue_name(queue_name), attempts: message.read_ct.to_i,
454
+ source_queue: source_queue || config.queue_name(queue_name), source_msg_id: message.msg_id.to_i,
455
+ source_enqueued_at: message.enqueued_at, attempts: message.read_ct.to_i,
446
456
  max_retries: config.max_retries, error: last_error
447
457
  )
448
458
  if source_queue
@@ -16,13 +16,21 @@ module Pgbus
16
16
  # once. Handing it a pre-serialized String stored a JSON *string*
17
17
  # (double-encoded), which every document reader of the column —
18
18
  # `payload->>'job_id'`, the job-class lookup, `scheduled_at` — misread.
19
+ #
20
+ # created_at is left to the column's CURRENT_TIMESTAMP default: the
21
+ # job's uniqueness lock is stamped by the database too, and discarding
22
+ # a parked job releases only a lock acquired no later than the park
23
+ # (UniquenessKey.release_if_unbound!), so both sides need one clock.
19
24
  def insert(concurrency_key:, queue_name:, payload:, duration:, priority: 0)
20
- Pgbus::BlockedExecution.create!(
21
- concurrency_key: concurrency_key,
22
- queue_name: queue_name,
23
- payload: payload,
24
- priority: priority,
25
- expires_at: Time.current + duration
25
+ Pgbus::BlockedExecution.insert!(
26
+ {
27
+ concurrency_key: concurrency_key,
28
+ queue_name: queue_name,
29
+ payload: payload,
30
+ priority: priority,
31
+ expires_at: Time.current + duration
32
+ },
33
+ record_timestamps: false
26
34
  )
27
35
  end
28
36
 
@@ -12,7 +12,7 @@ module Pgbus
12
12
  # config.workflow_max_payload_size = 131_072 # bytes: arguments and step outputs
13
13
  # config.workflow_error_backtrace_lines = 50 # backtrace lines kept on a failed step
14
14
  # config.workflow_waiting_visibility_threshold = 7.days # a step waiting this long without a
15
- # # wait_timeout: counts as stuck (waiting_overdue)
15
+ # # deadline counts as stuck (waiting_overdue)
16
16
  # config.workflow_subject_url = ->(type, id) { "/admin/#{type.underscore.pluralize}/#{id}" }
17
17
  # # the dashboard's subject link (nil: plain text)
18
18
  module WorkflowSettings
@@ -13,7 +13,8 @@ module Pgbus
13
13
  attr_accessor :database_url, :connection_params, :pool_size, :pool_timeout
14
14
 
15
15
  # Queue settings
16
- attr_accessor :default_queue, :queue_prefix
16
+ attr_accessor :default_queue
17
+ attr_reader :queue_prefix # rubocop:disable Style/AccessorGrouping -- the writer normalizes (see queue_prefix=)
17
18
 
18
19
  # Worker settings
19
20
  attr_accessor :polling_interval, :prefetch_limit, :execution_mode
@@ -597,6 +598,25 @@ module Pgbus
597
598
  @finalized = false
598
599
  end
599
600
 
601
+ # Stored the way the queue tables spell it ("my-app" → "my_app"). Table
602
+ # names are unchanged (queue_name normalizes the joined name the same
603
+ # way), but code that strips the prefix off a table name to get its queue
604
+ # back (the dashboard, the worker, a job's uniqueness lock queue) then
605
+ # finds it. A value normalize cannot spell (nothing table-safe left, or
606
+ # too long) is kept as given, as before: validate! rejects one that is
607
+ # too long, and queue_name drops one with nothing table-safe in it.
608
+ def queue_prefix=(value)
609
+ @queue_prefix = if value.is_a?(String) && !value.empty?
610
+ begin
611
+ QueueNameValidator.normalize(value)
612
+ rescue ArgumentError
613
+ value
614
+ end
615
+ else
616
+ value
617
+ end
618
+ end
619
+
600
620
  def queue_name(name)
601
621
  full = "#{queue_prefix}_#{name}"
602
622
  QueueNameValidator.normalize(full)
@@ -1002,6 +1022,27 @@ module Pgbus
1002
1022
  raise Pgbus::ConfigurationError, "capsule processes must be a positive Integer (got #{value.inspect})"
1003
1023
  end
1004
1024
 
1025
+ # A prefix with a leading, trailing or doubled underscore is kept as
1026
+ # written (its tables already exist under it), but queue_name collapses
1027
+ # the separator after it only for a queue name that needs normalizing:
1028
+ # "myapp_" gives myapp__orders yet myapp_orders_handler for
1029
+ # "orders-handler". A job's own release and the dashboard derive the same
1030
+ # lock queue for it and still match, but the reaper cannot resolve that
1031
+ # name back to the table, so a lock orphaned there is never reclaimed and
1032
+ # has to be released by key. Legal, but warned.
1033
+ def warn_queue_prefix_underscores
1034
+ return unless queue_prefix.match?(/\A_|_\z|__/)
1035
+
1036
+ suggested = queue_prefix.squeeze("_").delete_prefix("_").delete_suffix("_")
1037
+ suggestion = suggested.empty? ? "" : "; prefer #{suggested.inspect} for new deployments"
1038
+ Pgbus.logger.warn do
1039
+ "[Pgbus] queue_prefix #{queue_prefix.inspect} has a leading, trailing or doubled underscore — a queue " \
1040
+ "whose name needs normalizing (\"orders-handler\") gets a table spelled unlike its queue, so the " \
1041
+ "reaper cannot find that table: a uniqueness lock orphaned there (a worker crash before its release) " \
1042
+ "is never reclaimed and has to be released by key#{suggestion}"
1043
+ end
1044
+ end
1045
+
1005
1046
  # Buffered claims are kept invisible only by the visibility heartbeat.
1006
1047
  # Without it a message buffered longer than visibility_timeout can be
1007
1048
  # claimed by another worker while it still waits here. Legal, but warned.
@@ -1075,6 +1116,7 @@ module Pgbus
1075
1116
  { queue_prefix: queue_prefix, default_queue: default_queue }.each do |name, value|
1076
1117
  raise Pgbus::ConfigurationError, "#{name} must be a non-empty String" unless value.is_a?(String) && !value.empty?
1077
1118
  end
1119
+ warn_queue_prefix_underscores
1078
1120
 
1079
1121
  # web_auth gates the dashboard; a non-callable value silently leaves it
1080
1122
  # open (mirror streams_presence_member).
@@ -30,7 +30,13 @@ module Pgbus
30
30
 
31
31
  # error: FailedEventRecorder.last_error's Hash, or nil when no failure
32
32
  # was recorded. Returns the merged headers as a JSON string.
33
- def build(existing:, reason:, source:, source_queue:, attempts:, max_retries:, error: nil, now: Time.now.utc)
33
+ # source_msg_id / source_enqueued_at: the message's id in source_queue and
34
+ # when it was sent (the copy gets a new id and enqueued_at in the DLQ).
35
+ # A discard of the copy uses both to release only the uniqueness lock the
36
+ # original still holds: bound to it, or unbound and acquired before its
37
+ # send, never a lock it handed to a retry after (issue #564).
38
+ def build(existing:, reason:, source:, source_queue:, attempts:, max_retries:, error: nil, source_msg_id: nil,
39
+ source_enqueued_at: nil, now: Time.now.utc)
34
40
  headers = decode(existing, warn: true)
35
41
  retries = count(headers.delete(RETRIES_KEY))
36
42
  block = {
@@ -42,6 +48,8 @@ module Pgbus
42
48
  "max_retries" => max_retries,
43
49
  "dead_lettered_at" => iso(now)
44
50
  }
51
+ block["source_msg_id"] = source_msg_id if source_msg_id
52
+ block["source_enqueued_at"] = iso(source_enqueued_at) if source_enqueued_at
45
53
  block["retries_from_dlq"] = retries if retries.positive?
46
54
  block.merge!(error_fields(error)) if error
47
55
  JSON.generate(headers.merge(KEY => block))
@@ -79,9 +79,10 @@ module Pgbus
79
79
  # payload: workflow_class, workflow_id, name, duplicate,
80
80
  # woke (true when it woke the step waiting for it)
81
81
  # pgbus.workflow.wait_timed_out — a waiting step's wait_timeout: passed; its on_timeout
82
- # applied (the sweep, or TestHelpers#expire_wait)
82
+ # applied (the sweep, or TestHelpers#expire_wait); under
83
+ # notify the step keeps waiting, once per sweep that finds it late
83
84
  # payload: workflow_class, workflow_id, step_name, attempt,
84
- # action (pause! | cancel! | skip! | run)
85
+ # action (pause! | cancel! | skip! | run | notify)
85
86
  # pgbus.workflow.gauge — emitted by each dispatcher workflow sweep
86
87
  # payload: statuses ([{ workflow_class:, status:, count: }]),
87
88
  # stuck ([{ workflow_class:, reason:, count: }]), zero-filled
@@ -4,7 +4,7 @@
4
4
  ],
5
5
  "dashboard": {
6
6
  "title": "Pgbus",
7
- "description": "PostgreSQL-native job processing and event bus built on PGMQ.\nMonitor job throughput, queue depth, latency, dead letter queues,\nworker processes, event handlers, and stream broadcasts.\nWorkflow step outcomes and run times, and stuck workflows.\n",
7
+ "description": "PostgreSQL-native job processing and event bus built on PGMQ.\nMonitor job throughput, queue depth, latency, dead letter queues,\nworker processes, event handlers, and stream broadcasts.\nWorkflow step outcomes and run times, stuck workflows and late waits.\n",
8
8
  "visuals": [
9
9
  {
10
10
  "type": "timeseries",
@@ -520,6 +520,35 @@
520
520
  ]
521
521
  }
522
522
  ]
523
+ },
524
+ {
525
+ "type": "timeseries",
526
+ "display": "LINE",
527
+ "title": "Late workflow waits",
528
+ "description": "Deadlines an on_timeout: :notify wait missed, by class and step: the run keeps waiting for its signal.",
529
+ "line_label": "%workflow_class% - %step%",
530
+ "format": "number",
531
+ "draw_null_as_zero": true,
532
+ "metrics": [
533
+ {
534
+ "name": "pgbus_workflow_wait_overdue_total",
535
+ "fields": [
536
+ {
537
+ "field": "COUNTER"
538
+ }
539
+ ],
540
+ "tags": [
541
+ {
542
+ "key": "workflow_class",
543
+ "value": "*"
544
+ },
545
+ {
546
+ "key": "step",
547
+ "value": "*"
548
+ }
549
+ ]
550
+ }
551
+ ]
523
552
  }
524
553
  ]
525
554
  }
@@ -11,7 +11,7 @@ module Pgbus
11
11
  tool_name "pgbus_workflow_stats"
12
12
  title "Pgbus Workflow Stats"
13
13
  description <<~DESC
14
- One Pgbus::Workflow class over a window (1h, 24h default, 7d or 30d, capped at workflow_retention). steps, in declaration order (gravestones as kind "removed"; names only the rows hold as declared: false): counts (runs sitting at the step now by run status ready/performing/waiting/paused), run_ms (p50/p95/p99 of finished_at - started_at over completed or failed attempts) and pickup_ms (p50/p95 of scheduled → started), rates (failed: an error was recorded, reattempted, skipped) and outcomes over the sample_size attempts that finished in the window; percentiles over fewer than 20 samples are noisy. Also: started/finished/cancelled, e2e_ms, duration_histogram (finished runs per created → finished range), throughput (runs started/finished/cancelled per bucket_seconds), funnel (of the runs started in the window: reached = runs with a row for the step, stopped = where each run is now by status) and by_revision (the five app revisions that finished an attempt last, newest first, totals and per step).
14
+ One Pgbus::Workflow class over a window (1h, 24h default, 7d or 30d, capped at workflow_retention). steps, in declaration order (gravestones as kind "removed"; names only the rows hold as declared: false): wait_timeout (a wait_for: step's wait_timeout: in seconds, "dynamic" when a lambda or a method name sets it per run, null without one), counts (runs sitting at the step now by run status ready/performing/waiting/paused), run_ms (p50/p95/p99 of finished_at - started_at over completed or failed attempts) and pickup_ms (p50/p95 of scheduled → started), rates (failed: an error was recorded, reattempted, skipped) and outcomes over the sample_size attempts that finished in the window; percentiles over fewer than 20 samples are noisy. Also: started/finished/cancelled, e2e_ms, duration_histogram (finished runs per created → finished range), throughput (runs started/finished/cancelled per bucket_seconds), funnel (of the runs started in the window: reached = runs with a row for the step, stopped = where each run is now by status) and by_revision (the five app revisions that finished an attempt last, newest first, totals and per step).
15
15
  DESC
16
16
 
17
17
  input_schema(
@@ -9,7 +9,7 @@ module Pgbus
9
9
  tool_name "pgbus_workflows_attention"
10
10
  title "Pgbus Workflows Needing Attention"
11
11
  description <<~DESC
12
- Ongoing Pgbus::Workflow runs that need a person, grouped by (workflow_class, step_name, reason, error_class): reason is paused_error, stalled, paused_manual, overdue (due but not picked up past workflow_stall_threshold) or waiting_overdue (waiting past workflow_waiting_visibility_threshold without a wait_timeout). Each group has its run count, first_seen_at/last_seen_at, the status its runs are in and the actions that status allows; total_runs sums the groups (at most 200 groups, biggest first). workflow_class narrows them to one class. The newest error message is redacted unless the server allows payloads and include_payloads is set.
12
+ Ongoing Pgbus::Workflow runs that need a person, grouped by (workflow_class, step_name, reason, error_class): reason is paused_error, stalled, paused_manual, overdue (due but not picked up past workflow_stall_threshold) or waiting_overdue (waiting past workflow_waiting_visibility_threshold without a deadline: no wait_timeout, or a per-run one that set none or cleared it after a :notify notice). Each group has its run count, first_seen_at/last_seen_at, the status its runs are in and the actions that status allows; total_runs sums the groups (at most 200 groups, biggest first). workflow_class narrows them to one class. The newest error message is redacted unless the server allows payloads and include_payloads is set.
13
13
  DESC
14
14
 
15
15
  input_schema(
@@ -12,6 +12,7 @@ module Pgbus
12
12
  # pgbus_workflows_paused_total{workflow_class, error_class} counter (error_class absent for a manual pause)
13
13
  # pgbus_workflows_cancelled_total{workflow_class} counter
14
14
  # pgbus_workflow_stalls_total{workflow_class, step, detection} counter (detection: redelivery | sweep)
15
+ # pgbus_workflow_wait_overdue_total{workflow_class, step} counter (a late notice of an on_timeout: :notify wait)
15
16
  # pgbus_workflows{workflow_class, status} gauge, from each sweep
16
17
  # pgbus_workflows_stuck{workflow_class, reason} gauge, from each sweep
17
18
  #
@@ -20,7 +21,7 @@ module Pgbus
20
21
  module WorkflowMetrics
21
22
  EVENTS = %w[
22
23
  pgbus.workflow.step pgbus.workflow_finished pgbus.workflow_paused pgbus.workflow_cancelled
23
- pgbus.workflow.stall_detected pgbus.workflow.gauge
24
+ pgbus.workflow.stall_detected pgbus.workflow.wait_timed_out pgbus.workflow.gauge
24
25
  ].freeze
25
26
 
26
27
  module_function
@@ -37,6 +38,7 @@ module Pgbus
37
38
  when "pgbus.workflow_cancelled" then [counter("pgbus_workflows_cancelled_total", workflow_class: klass)]
38
39
  when "pgbus.workflow.stall_detected"
39
40
  [counter("pgbus_workflow_stalls_total", workflow_class: klass, step: payload[:step_name], detection: payload[:detection])]
41
+ when "pgbus.workflow.wait_timed_out" then wait_timed_out_samples(klass, payload)
40
42
  when "pgbus.workflow.gauge" then gauge_samples(payload)
41
43
  else []
42
44
  end
@@ -49,6 +51,14 @@ module Pgbus
49
51
  samples
50
52
  end
51
53
 
54
+ # Only :notify keeps waiting past the deadline; every other on_timeout
55
+ # ends the wait and is counted by the step or run it ends.
56
+ def wait_timed_out_samples(klass, payload)
57
+ return [] unless payload[:action] == "notify"
58
+
59
+ [counter("pgbus_workflow_wait_overdue_total", workflow_class: klass, step: payload[:step_name])]
60
+ end
61
+
52
62
  def gauge_samples(payload)
53
63
  statuses = Array(payload[:statuses]).map do |row|
54
64
  [:gauge, "pgbus_workflows", row[:count], { workflow_class: row[:workflow_class], status: row[:status] }]
@@ -296,7 +296,8 @@ module Pgbus
296
296
  last_error = FailedEventRecorder.last_error(queue_name: queue_name, msg_id: message.msg_id.to_i)
297
297
  headers = DeadLetterHeader.build(
298
298
  existing: message.headers, reason: DeadLetterHeader::REASON_MAX_RETRIES, source: "consumer",
299
- source_queue: config.queue_name(queue_name), attempts: message.read_ct.to_i,
299
+ source_queue: config.queue_name(queue_name), source_msg_id: message.msg_id.to_i,
300
+ source_enqueued_at: message.enqueued_at, attempts: message.read_ct.to_i,
300
301
  max_retries: config.max_retries, error: last_error
301
302
  )
302
303
  Pgbus.client.move_to_dead_letter(queue_name, message, headers: headers)
@@ -505,12 +505,22 @@ module Pgbus
505
505
  # Unbound rows (synthetic "pending" queue and/or msg_id=0) must not probe
506
506
  # pgmq.q_<prefix>_pending — that table does not exist, message_exists?
507
507
  # returns nil, and the reaper would keep the lock forever (issue #418).
508
- # Scan every live queue for the payload key instead.
508
+ # Scan every live queue for the payload key instead. A job parked behind
509
+ # its concurrency slot is in no queue: its key counts as live while it
510
+ # is parked (issue #564). The parked jobs are asked first: a promotion
511
+ # sends the job's message before its parked row's delete commits (or
512
+ # both in one transaction), so a job promoted between the two reads is
513
+ # still parked for the first or already queued for the second. Read the
514
+ # other way round it could be in neither.
509
515
  def gone_unbound_locks(unbound)
510
516
  return [] if unbound.empty?
511
517
 
512
- present = Pgbus.client.uniqueness_keys_present(unbound.map(&:lock_key))
513
- unbound.reject { |key| present.include?(key.lock_key) }
518
+ parked = BlockedExecution.parked_uniqueness_keys(unbound.map(&:lock_key))
519
+ unparked = unbound.reject { |key| parked.include?(key.lock_key) }
520
+ return [] if unparked.empty?
521
+
522
+ present = Pgbus.client.uniqueness_keys_present(unparked.map(&:lock_key))
523
+ unparked.reject { |key| present.include?(key.lock_key) }
514
524
  end
515
525
 
516
526
  # Returns true if the message referenced by this bound lock is definitely
@@ -407,12 +407,21 @@ module Pgbus
407
407
  # so we also pass `limit: qty` to cap the total across all queues —
408
408
  # otherwise we get `queue_count * qty` messages and overflow the
409
409
  # execution pool, crashing the worker fork (issue #123).
410
+ #
411
+ # The logical name is the configured one the table was made from, not
412
+ # the table name with its prefix cut: a name normalized for its table
413
+ # ("orders-handler" → "orders_handler") would otherwise reach the
414
+ # executor spelled unlike the name its jobs' uniqueness locks are bound
415
+ # under, and unlike every other fetch path (issue #564).
410
416
  def fetch_multi(active_queues, qty)
411
417
  messages = Pgbus.client.read_multi(active_queues, qty: qty, limit: qty) || []
418
+ return messages if messages.empty?
419
+
412
420
  prefix = "#{config.queue_prefix}_"
421
+ configured = active_queues.to_h { |q| [config.queue_name(q), q] }
413
422
 
414
423
  messages.map do |m|
415
- logical = m.queue_name&.delete_prefix(prefix) || active_queues.first
424
+ logical = configured[m.queue_name] || m.queue_name&.delete_prefix(prefix) || active_queues.first
416
425
  [logical, m]
417
426
  end
418
427
  end
@@ -143,7 +143,7 @@ module Pgbus
143
143
  key = resolve_uniqueness_key(job_class, config, task)
144
144
  return nil unless key
145
145
 
146
- acquired = UniquenessKey.acquire!(key, queue_name: resolve_queue(task), msg_id: 0)
146
+ acquired = UniquenessKey.acquire!(key, queue_name: Uniqueness.lock_queue_name(resolve_queue(task)), msg_id: 0)
147
147
 
148
148
  if acquired
149
149
  key
@@ -72,6 +72,7 @@ module Pgbus
72
72
  @schema_mutex = Mutex.new
73
73
  @seats_supported = nil
74
74
  @probed_at = nil
75
+ @schema_generation = 0
75
76
 
76
77
  class << self
77
78
  # True when pgbus_presence_members has the seat columns (the
@@ -81,17 +82,29 @@ module Pgbus
81
82
  # The answer is read and written under @schema_mutex only (issue
82
83
  # #527). A successful probe is kept as above; a probe that raises is
83
84
  # not, so the next call probes again.
85
+ #
86
+ # The probe itself runs outside @schema_mutex (issue #566): it waits
87
+ # for the adapter's lock, and a request inside a transaction holds
88
+ # that lock while it asks seats_supported?. Two threads may probe at
89
+ # once; a true answer one of them stored is never taken back, and a
90
+ # probe that a reset_schema_cache! overtook stores nothing.
84
91
  def seats_supported?
92
+ cached, generation = @schema_mutex.synchronize do
93
+ [(@seats_supported if schema_answer_fresh?(@seats_supported, @probed_at)), @schema_generation]
94
+ end
95
+ return cached unless cached.nil?
96
+
97
+ rows = exec_params(<<~SQL, [])
98
+ SELECT 1 AS present FROM information_schema.columns
99
+ WHERE table_name = 'pgbus_presence_members' AND column_name = 'connection_id'
100
+ AND table_schema = ANY (current_schemas(false))
101
+ SQL
102
+ supported = !rows.to_a.empty?
85
103
  @schema_mutex.synchronize do
86
- next @seats_supported if schema_answer_fresh?(@seats_supported, @probed_at)
104
+ next supported unless generation == @schema_generation
87
105
 
88
- rows = connection.exec_params(<<~SQL, [])
89
- SELECT 1 AS present FROM information_schema.columns
90
- WHERE table_name = 'pgbus_presence_members' AND column_name = 'connection_id'
91
- AND table_schema = ANY (current_schemas(false))
92
- SQL
93
106
  @probed_at = monotonic_now
94
- @seats_supported = !rows.to_a.empty?
107
+ @seats_supported ||= supported
95
108
  end
96
109
  end
97
110
 
@@ -108,6 +121,7 @@ module Pgbus
108
121
 
109
122
  def reset_schema_cache!
110
123
  @schema_mutex.synchronize do
124
+ @schema_generation += 1
111
125
  @probed_at = nil
112
126
  @seats_supported = nil
113
127
  end
@@ -124,7 +138,7 @@ module Pgbus
124
138
  def expire_seats!(ttl_for:, min_ttl:)
125
139
  return [] unless seats_supported?
126
140
 
127
- candidates = connection.exec_params(<<~SQL, [min_ttl])
141
+ candidates = exec_params(<<~SQL, [min_ttl])
128
142
  SELECT stream_name, connection_id FROM pgbus_presence_members
129
143
  WHERE connection_id IS NOT NULL AND last_seen_at < NOW() - make_interval(secs => $1)
130
144
  SQL
@@ -133,23 +147,23 @@ module Pgbus
133
147
  end
134
148
  end
135
149
 
150
+ # The PG::Connection under the adapter. Run SQL through exec_params,
151
+ # not on this: the adapter's lock is released once it is returned.
136
152
  def connection
137
- # Respect multi-database setups (connects_to). When
138
- # Pgbus.configuration.connects_to is set, the pgbus tables
139
- # live in a separate database accessed via Pgbus::BusRecord;
140
- # the default path uses ActiveRecord::Base. Matches the
141
- # canonical pattern in Pgbus::Process::QueueLock#connection
142
- # and Pgbus::Configuration's connection probe.
143
- unless defined?(::ActiveRecord::Base)
144
- raise Pgbus::ConfigurationError,
145
- "Pgbus::Streams::Presence requires ActiveRecord (no AR connection available)"
146
- end
153
+ adapter.raw_connection
154
+ end
147
155
 
148
- if Pgbus.configuration.connects_to
149
- Pgbus::BusRecord.connection.raw_connection
150
- else
151
- ::ActiveRecord::Base.connection.raw_connection
152
- end
156
+ # Runs one statement on the adapter's PG::Connection while holding
157
+ # the adapter's lock (issue #566). raw_connection takes that lock
158
+ # only to hand the connection out; on a pinned pool (transactional
159
+ # tests) every thread shares the one adapter, and its lock is all
160
+ # that keeps a second thread's statement off the libpq connection
161
+ # while this one is in flight. Unpinned, the lock is a NullLock. The
162
+ # lock is reentrant, so a thread already holding it (a request
163
+ # inside a transaction) re-enters it.
164
+ def exec_params(sql, binds)
165
+ adapter = self.adapter
166
+ adapter.lock.synchronize { adapter.raw_connection.exec_params(sql, binds) }
153
167
  end
154
168
 
155
169
  # Reports a seat join/leave to streams_presence_on_change (issue #515).
@@ -171,6 +185,21 @@ module Pgbus
171
185
 
172
186
  private
173
187
 
188
+ # Respect multi-database setups (connects_to). When
189
+ # Pgbus.configuration.connects_to is set, the pgbus tables
190
+ # live in a separate database accessed via Pgbus::BusRecord;
191
+ # the default path uses ActiveRecord::Base. Matches the
192
+ # canonical pattern in Pgbus::Process::QueueLock#connection
193
+ # and Pgbus::Configuration's connection probe.
194
+ def adapter
195
+ unless defined?(::ActiveRecord::Base)
196
+ raise Pgbus::ConfigurationError,
197
+ "Pgbus::Streams::Presence requires ActiveRecord (no AR connection available)"
198
+ end
199
+
200
+ Pgbus.configuration.connects_to ? Pgbus::BusRecord.connection : ::ActiveRecord::Base.connection
201
+ end
202
+
174
203
  def schema_answer_fresh?(supported, probed_at)
175
204
  return false if supported.nil? || probed_at.nil?
176
205
 
@@ -178,7 +207,7 @@ module Pgbus
178
207
  end
179
208
 
180
209
  def expire_stream_seats(stream_name, connection_ids, ttl)
181
- rows = connection.exec_params(<<~SQL, [stream_name, text_array(connection_ids), ttl])
210
+ rows = exec_params(<<~SQL, [stream_name, text_array(connection_ids), ttl])
182
211
  DELETE FROM pgbus_presence_members
183
212
  WHERE stream_name = $1 AND connection_id = ANY($2::text[])
184
213
  AND last_seen_at < NOW() - make_interval(secs => $3)
@@ -306,7 +335,7 @@ module Pgbus
306
335
  return touch_seats(Array(connection_id)) if connection_id
307
336
 
308
337
  member_id = member_id.to_s
309
- connection.exec_params(<<~SQL, [@stream.name, member_id])
338
+ exec_params(<<~SQL, [@stream.name, member_id])
310
339
  UPDATE pgbus_presence_members
311
340
  SET last_seen_at = NOW()
312
341
  WHERE stream_name = $1 AND member_id = $2#{" AND connection_id IS NULL" if seats_now?}
@@ -320,7 +349,7 @@ module Pgbus
320
349
  def set_state(connection_id:, state:)
321
350
  raise ArgumentError, "set_state takes a Hash (got #{state.class})" unless state.is_a?(Hash)
322
351
 
323
- rows = connection.exec_params(<<~SQL, [@stream.name, connection_id.to_s, JSON.generate(state)])
352
+ rows = exec_params(<<~SQL, [@stream.name, connection_id.to_s, JSON.generate(state)])
324
353
  UPDATE pgbus_presence_members
325
354
  SET state = state || $3::jsonb, last_seen_at = NOW()
326
355
  WHERE stream_name = $1 AND connection_id = $2
@@ -335,7 +364,7 @@ module Pgbus
335
364
  # table each row is one seat and also carries connection_id, seat
336
365
  # and state; group by "id" for one entry per member.
337
366
  def members
338
- rows = connection.exec_params(<<~SQL, [@stream.name])
367
+ rows = exec_params(<<~SQL, [@stream.name])
339
368
  SELECT #{seats? ? SEAT_RECORD_COLUMNS : RECORD_COLUMNS}
340
369
  FROM pgbus_presence_members
341
370
  WHERE stream_name = $1
@@ -347,7 +376,7 @@ module Pgbus
347
376
  # Returns the count of current members. Faster than members.size
348
377
  # because it doesn't deserialize metadata.
349
378
  def count
350
- rows = connection.exec_params(<<~SQL, [@stream.name])
379
+ rows = exec_params(<<~SQL, [@stream.name])
351
380
  SELECT COUNT(*) AS n FROM pgbus_presence_members WHERE stream_name = $1
352
381
  SQL
353
382
  rows.first["n"].to_i
@@ -362,7 +391,7 @@ module Pgbus
362
391
  # workers running the sweep concurrently won't double-emit leave
363
392
  # events. Yields each expired record when a block is given.
364
393
  def sweep!(older_than:)
365
- rows = connection.exec_params(<<~SQL, [@stream.name, older_than])
394
+ rows = exec_params(<<~SQL, [@stream.name, older_than])
366
395
  DELETE FROM pgbus_presence_members
367
396
  WHERE stream_name = $1 AND last_seen_at < $2#{" AND connection_id IS NULL" if seats_now?}
368
397
  RETURNING #{RECORD_COLUMNS}
@@ -388,7 +417,7 @@ module Pgbus
388
417
 
389
418
  def upsert_member_once(member_id, metadata)
390
419
  conflict = seats? ? "(stream_name, member_id) WHERE connection_id IS NULL" : "(stream_name, member_id)"
391
- rows = connection.exec_params(<<~SQL, [@stream.name, member_id, JSON.generate(metadata)])
420
+ rows = exec_params(<<~SQL, [@stream.name, member_id, JSON.generate(metadata)])
392
421
  INSERT INTO pgbus_presence_members (stream_name, member_id, metadata, joined_at, last_seen_at)
393
422
  VALUES ($1, $2, $3::jsonb, NOW(), NOW())
394
423
  ON CONFLICT #{conflict}
@@ -399,7 +428,7 @@ module Pgbus
399
428
  end
400
429
 
401
430
  def upsert_seat(member_id, metadata, connection_id, seat)
402
- rows = connection.exec_params(<<~SQL, [@stream.name, member_id, JSON.generate(metadata), connection_id, seat])
431
+ rows = exec_params(<<~SQL, [@stream.name, member_id, JSON.generate(metadata), connection_id, seat])
403
432
  INSERT INTO pgbus_presence_members
404
433
  (stream_name, member_id, metadata, connection_id, seat, joined_at, last_seen_at)
405
434
  VALUES ($1, $2, $3::jsonb, $4, $5, NOW(), NOW())
@@ -412,7 +441,7 @@ module Pgbus
412
441
  end
413
442
 
414
443
  def delete_member(member_id)
415
- rows = connection.exec_params(<<~SQL, [@stream.name, member_id])
444
+ rows = exec_params(<<~SQL, [@stream.name, member_id])
416
445
  DELETE FROM pgbus_presence_members
417
446
  WHERE stream_name = $1 AND member_id = $2#{" AND connection_id IS NULL" if seats_now?}
418
447
  RETURNING #{RECORD_COLUMNS}
@@ -422,7 +451,7 @@ module Pgbus
422
451
  end
423
452
 
424
453
  def delete_seat(connection_id)
425
- rows = connection.exec_params(<<~SQL, [@stream.name, connection_id])
454
+ rows = exec_params(<<~SQL, [@stream.name, connection_id])
426
455
  DELETE FROM pgbus_presence_members
427
456
  WHERE stream_name = $1 AND connection_id = $2
428
457
  RETURNING #{SEAT_RECORD_COLUMNS}
@@ -434,7 +463,7 @@ module Pgbus
434
463
  def touch_seats(connection_ids)
435
464
  return [] if connection_ids.empty?
436
465
 
437
- rows = connection.exec_params(<<~SQL, [@stream.name, self.class.text_array(connection_ids)])
466
+ rows = exec_params(<<~SQL, [@stream.name, self.class.text_array(connection_ids)])
438
467
  UPDATE pgbus_presence_members
439
468
  SET last_seen_at = NOW()
440
469
  WHERE stream_name = $1 AND connection_id = ANY($2::text[])
@@ -443,8 +472,8 @@ module Pgbus
443
472
  rows.to_a.map { |row| row["connection_id"] }
444
473
  end
445
474
 
446
- def connection
447
- self.class.connection
475
+ def exec_params(sql, binds)
476
+ self.class.exec_params(sql, binds)
448
477
  end
449
478
  end
450
479
  end