pgbus 0.20.1 → 0.21.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +66 -8
  3. data/README.md +1 -1
  4. data/app/helpers/pgbus/workflow_insights_helper.rb +8 -0
  5. data/app/models/pgbus/workflow_entry.rb +3 -2
  6. data/app/views/pgbus/workflows/definitions/_step_card.html.erb +3 -0
  7. data/config/locales/da.yml +3 -0
  8. data/config/locales/de.yml +3 -0
  9. data/config/locales/en.yml +3 -0
  10. data/config/locales/es.yml +3 -0
  11. data/config/locales/fi.yml +3 -0
  12. data/config/locales/fr.yml +3 -0
  13. data/config/locales/it.yml +3 -0
  14. data/config/locales/ja.yml +3 -0
  15. data/config/locales/nb.yml +3 -0
  16. data/config/locales/nl.yml +3 -0
  17. data/config/locales/pt.yml +3 -0
  18. data/config/locales/sv.yml +3 -0
  19. data/lib/generators/pgbus/templates/add_workflows.rb.erb +3 -1
  20. data/lib/generators/pgbus/templates/migration.rb.erb +3 -1
  21. data/lib/pgbus/batch/sweep.rb +10 -5
  22. data/lib/pgbus/client.rb +109 -42
  23. data/lib/pgbus/configuration/workflow_settings.rb +1 -1
  24. data/lib/pgbus/mcp/tools/workflow_stats_tool.rb +1 -1
  25. data/lib/pgbus/mcp/tools/workflows_attention_tool.rb +1 -1
  26. data/lib/pgbus/version.rb +1 -1
  27. data/lib/pgbus/web/data_source/workflow_list.rb +6 -4
  28. data/lib/pgbus/web/data_source.rb +39 -22
  29. data/lib/pgbus/web/workflow_insights_report.rb +11 -3
  30. data/lib/pgbus/web/workflow_state.rb +5 -2
  31. data/lib/pgbus/workflow/instance.rb +36 -3
  32. data/lib/pgbus/workflow/runner.rb +2 -2
  33. data/lib/pgbus/workflow/signal_peek.rb +47 -0
  34. data/lib/pgbus/workflow/signals.rb +31 -10
  35. data/lib/pgbus/workflow/step_definition.rb +45 -3
  36. data/lib/pgbus/workflow/step_execution.rb +27 -10
  37. data/lib/pgbus/workflow/wait_timeout.rb +38 -7
  38. data/lib/pgbus/workflow.rb +11 -5
  39. data/sig/pgbus/client.rbs +7 -5
  40. data/sig/pgbus.rbs +43 -0
  41. metadata +2 -1
data/lib/pgbus/client.rb CHANGED
@@ -720,16 +720,24 @@ module Pgbus
720
720
  # reaper to determine if a uniqueness lock with msg_id=0 (placeholder)
721
721
  # still has a corresponding queue message.
722
722
  #
723
- # +queue_name+ may be either a logical name (e.g. "default") or an already
724
- # prefixed physical name (e.g. "pgbus_default"). The client normalizes both.
723
+ # +queue_name+ is a logical name (e.g. "default"), expanded through the
724
+ # queue strategy, unless the caller passes +prefixed: false+ for a physical
725
+ # table name (e.g. "pgbus_default_p1"), used as is. The name's spelling
726
+ # decides nothing: a logical "pgbus_mailers" lives in pgbus_pgbus_mailers
727
+ # (issue #569).
725
728
  #
726
729
  # Returns:
727
730
  # true — the message definitely exists in the queue
728
731
  # false — the message definitely does not exist
729
732
  # nil — could not determine (e.g. queue table missing or unknown error).
730
733
  # Callers MUST treat nil as "exists" for safety.
731
- def message_in_queue?(queue_name, msg_id:)
732
- message_exists?(queue_name, msg_id: msg_id)
734
+ #
735
+ # Under priority routing a logical name probes every `_pN` table, and each
736
+ # table has its own msg_id sequence: a bare +msg_id+ may match a different
737
+ # message in a sibling sub-queue. A caller that knows the sub-queue passes
738
+ # its physical name with +prefixed: false+.
739
+ def message_in_queue?(queue_name, msg_id:, prefixed: true)
740
+ message_exists?(queue_name, msg_id: msg_id, prefixed: prefixed)
733
741
  end
734
742
 
735
743
  # Look up a queue row by ActiveJob job_id in the JSON payload. Used by the
@@ -738,10 +746,10 @@ module Pgbus
738
746
  #
739
747
  # Same tri-state as message_exists?: true / false / nil (unknown). A
740
748
  # logical name expands to every physical table the strategy owns (the
741
- # `_pN` sub-queues under priority routing); an already-prefixed physical
742
- # name probes itself only.
743
- def message_with_job_id?(queue_name, job_id:)
744
- tables = lookup_physical_queue_names(queue_name).map { |name| QueueNameValidator.sanitize!(name) }
749
+ # `_pN` sub-queues under priority routing); a physical name passed with
750
+ # +prefixed: false+ probes itself only.
751
+ def message_with_job_id?(queue_name, job_id:, prefixed: true)
752
+ tables = lookup_physical_queue_names(queue_name, prefixed: prefixed).map { |name| QueueNameValidator.sanitize!(name) }
745
753
  determined = false
746
754
 
747
755
  synchronized do
@@ -758,43 +766,85 @@ module Pgbus
758
766
  determined ? false : nil # rubocop:disable Style/ReturnNilInPredicateMethodDefinition -- tri-state: nil means unknown
759
767
  end
760
768
 
761
- # DLQ companion of a logical or already-physical queue name. Does not
762
- # re-prefix a name that resolve_full_queue_name already expanded.
763
- def dead_letter_physical_name(queue_name)
764
- full = resolve_full_queue_name(queue_name)
769
+ # The physical table that holds message +msg_id+, or nil when none does.
770
+ # See locate_messages.
771
+ def locate_message(queue_name, msg_id:, job_id: nil, prefixed: true)
772
+ locate_messages(queue_name, [[msg_id, job_id]], prefixed: prefixed)[msg_id.to_i]
773
+ end
774
+
775
+ # { msg_id => physical table } for each [msg_id, job_id] pair whose
776
+ # message is queued, one query per table on one connection. A logical
777
+ # name expands to every table the strategy owns, and under priority
778
+ # routing each `_pN` sub-queue numbers its messages from its own
779
+ # sequence: only a message carrying the pair's job_id matches, or any
780
+ # message with that msg_id when job_id is nil, the first table winning
781
+ # (issue #577). A physical name with +prefixed: false+ is probed alone.
782
+ # A table that does not exist holds nothing; other errors raise.
783
+ def locate_messages(queue_name, messages, prefixed: true)
784
+ wanted = messages.to_h { |msg_id, job_id| [msg_id.to_i, job_id&.to_s] }
785
+ tables = lookup_physical_queue_names(queue_name, prefixed: prefixed)
786
+ located = {}
787
+
788
+ synchronized do
789
+ with_raw_connection do |conn|
790
+ tables.each do |name|
791
+ pending = wanted.keys - located.keys
792
+ break if pending.empty?
793
+
794
+ queued_job_ids(conn, QueueNameValidator.sanitize!(name), pending).each do |row|
795
+ msg_id = row["msg_id"].to_i
796
+ job_id = wanted[msg_id]
797
+ located[msg_id] ||= name if job_id.nil? || row["job_id"] == job_id
798
+ end
799
+ end
800
+ end
801
+ end
802
+
803
+ located
804
+ end
805
+
806
+ # DLQ companion of a logical queue name, or of a physical table name passed
807
+ # with +prefixed: false+.
808
+ def dead_letter_physical_name(queue_name, prefixed: true)
809
+ return config.dead_letter_queue_name(queue_name) if prefixed
810
+
765
811
  # Priority sub-queues share the logical queue's DLQ (`pgbus_default_dlq`),
766
- # not a per-priority `…_pN_dlq`. Only strip when the strategy actually
767
- # creates `_pN` tables — a logical queue named `orders_p0` must keep it.
768
- base = @queue_strategy.priority? ? full.sub(/_p\d+\z/, "") : full
812
+ # not a per-priority `…_pN_dlq`. Only a physical name can be a sub-queue,
813
+ # and only when the strategy actually creates `_pN` tables: a logical
814
+ # queue named `orders_p0` keeps its suffix.
815
+ name = queue_name.to_s
816
+ base = @queue_strategy.priority? ? name.sub(/_p\d+\z/, "") : name
769
817
  "#{base}#{Pgbus::DEAD_LETTER_SUFFIX}"
770
818
  end
771
819
 
772
820
  # Same tri-state as message_exists?: true / false / nil (unknown).
773
- def message_archived?(queue_name, msg_id:)
774
- full_name = resolve_full_queue_name(queue_name)
775
- sanitized = QueueNameValidator.sanitize!(full_name)
821
+ # A logical name probes the archive of every physical table the strategy
822
+ # owns, like message_exists?; a physical name (prefixed: false) its own.
823
+ # A bare msg_id is unique per table only (see message_in_queue?).
824
+ def message_archived?(queue_name, msg_id:, prefixed: true)
825
+ tables = lookup_physical_queue_names(queue_name, prefixed: prefixed).map { |name| QueueNameValidator.sanitize!(name) }
826
+ determined = false
776
827
 
777
828
  synchronized do
778
829
  with_raw_connection do |conn|
779
- msg_id_archived?(conn, sanitized, msg_id.to_i)
830
+ tables.each do |sanitized|
831
+ archived = probe_archive_presence(conn, sanitized, msg_id.to_i)
832
+ return true if archived == true
833
+
834
+ determined = true unless archived == :missing
835
+ end
780
836
  end
781
837
  end
782
- rescue ActiveRecord::StatementInvalid => e
783
- raise unless undefined_table_error?(e)
784
-
785
- nil
786
- rescue StandardError => e
787
- raise unless defined?(PG::UndefinedTable) && e.is_a?(PG::UndefinedTable)
788
838
 
789
- nil
839
+ determined ? false : nil # rubocop:disable Style/ReturnNilInPredicateMethodDefinition -- tri-state: nil means unknown
790
840
  end
791
841
 
792
- def message_exists?(queue_name, msg_id: nil, uniqueness_key: nil)
842
+ def message_exists?(queue_name, msg_id: nil, uniqueness_key: nil, prefixed: true)
793
843
  has_msg_id = !msg_id.nil?
794
844
  has_uniqueness_key = !uniqueness_key.nil?
795
845
  raise ArgumentError, "pass msg_id, uniqueness_key, or both" unless has_msg_id || has_uniqueness_key
796
846
 
797
- tables = lookup_physical_queue_names(queue_name).map { |name| QueueNameValidator.sanitize!(name) }
847
+ tables = lookup_physical_queue_names(queue_name, prefixed: prefixed).map { |name| QueueNameValidator.sanitize!(name) }
798
848
  determined = false
799
849
 
800
850
  synchronized do
@@ -1092,21 +1142,10 @@ module Pgbus
1092
1142
  end
1093
1143
  end
1094
1144
 
1095
- # Accept either a logical name ("default") or an already-prefixed
1096
- # physical name ("pgbus_default") and return the physical name.
1097
- # Coerces symbols to strings so callers can pass either form.
1098
- def resolve_full_queue_name(queue_name)
1099
- name = queue_name.to_s
1100
- prefix = "#{config.queue_prefix}_"
1101
- name.start_with?(prefix) ? name : config.queue_name(name)
1102
- end
1103
-
1104
1145
  # Logical names expand through QueueFactory so a priority queue's _pN
1105
- # tables are included; already-prefixed physical names stay as-is.
1106
- def lookup_physical_queue_names(queue_name)
1107
- name = queue_name.to_s
1108
- prefix = "#{config.queue_prefix}_"
1109
- name.start_with?(prefix) ? [name] : @queue_strategy.physical_queue_names(name)
1146
+ # tables are included; a physical name (prefixed: false) stays as is.
1147
+ def lookup_physical_queue_names(queue_name, prefixed:)
1148
+ prefixed ? @queue_strategy.physical_queue_names(queue_name.to_s) : [queue_name.to_s]
1110
1149
  end
1111
1150
 
1112
1151
  def probe_queue_presence(conn, sanitized, msg_id:, uniqueness_key:)
@@ -1125,6 +1164,18 @@ module Pgbus
1125
1164
  :missing
1126
1165
  end
1127
1166
 
1167
+ def probe_archive_presence(conn, sanitized, msg_id)
1168
+ msg_id_archived?(conn, sanitized, msg_id)
1169
+ rescue ActiveRecord::StatementInvalid => e
1170
+ raise unless undefined_table_error?(e)
1171
+
1172
+ :missing
1173
+ rescue StandardError => e
1174
+ raise unless defined?(PG::UndefinedTable) && e.is_a?(PG::UndefinedTable)
1175
+
1176
+ :missing
1177
+ end
1178
+
1128
1179
  def probe_job_id_presence(conn, sanitized, job_id)
1129
1180
  job_id_present?(conn, sanitized, job_id)
1130
1181
  rescue ActiveRecord::StatementInvalid => e
@@ -1137,6 +1188,22 @@ module Pgbus
1137
1188
  :missing
1138
1189
  end
1139
1190
 
1191
+ # msg_ids are integers, so the array literal is safe.
1192
+ def queued_job_ids(conn, sanitized, msg_ids)
1193
+ conn.exec_params(
1194
+ "SELECT msg_id, message::jsonb ->> 'job_id' AS job_id FROM pgmq.q_#{sanitized} WHERE msg_id = ANY($1::bigint[])",
1195
+ ["{#{msg_ids.join(",")}}"]
1196
+ )
1197
+ rescue ActiveRecord::StatementInvalid => e
1198
+ raise unless undefined_table_error?(e)
1199
+
1200
+ []
1201
+ rescue StandardError => e
1202
+ raise unless defined?(PG::UndefinedTable) && e.is_a?(PG::UndefinedTable)
1203
+
1204
+ []
1205
+ end
1206
+
1140
1207
  def scan_uniqueness_keys(conn, sanitized, keys, found)
1141
1208
  placeholders = keys.each_index.map { |i| "$#{i + 1}" }.join(", ")
1142
1209
  result = conn.exec_params(
@@ -12,7 +12,7 @@ module Pgbus
12
12
  # config.workflow_max_payload_size = 131_072 # bytes: arguments and step outputs
13
13
  # config.workflow_error_backtrace_lines = 50 # backtrace lines kept on a failed step
14
14
  # config.workflow_waiting_visibility_threshold = 7.days # a step waiting this long without a
15
- # # wait_timeout: counts as stuck (waiting_overdue)
15
+ # # deadline counts as stuck (waiting_overdue)
16
16
  # config.workflow_subject_url = ->(type, id) { "/admin/#{type.underscore.pluralize}/#{id}" }
17
17
  # # the dashboard's subject link (nil: plain text)
18
18
  module WorkflowSettings
@@ -11,7 +11,7 @@ module Pgbus
11
11
  tool_name "pgbus_workflow_stats"
12
12
  title "Pgbus Workflow Stats"
13
13
  description <<~DESC
14
- One Pgbus::Workflow class over a window (1h, 24h default, 7d or 30d, capped at workflow_retention). steps, in declaration order (gravestones as kind "removed"; names only the rows hold as declared: false): counts (runs sitting at the step now by run status ready/performing/waiting/paused), run_ms (p50/p95/p99 of finished_at - started_at over completed or failed attempts) and pickup_ms (p50/p95 of scheduled → started), rates (failed: an error was recorded, reattempted, skipped) and outcomes over the sample_size attempts that finished in the window; percentiles over fewer than 20 samples are noisy. Also: started/finished/cancelled, e2e_ms, duration_histogram (finished runs per created → finished range), throughput (runs started/finished/cancelled per bucket_seconds), funnel (of the runs started in the window: reached = runs with a row for the step, stopped = where each run is now by status) and by_revision (the five app revisions that finished an attempt last, newest first, totals and per step).
14
+ One Pgbus::Workflow class over a window (1h, 24h default, 7d or 30d, capped at workflow_retention). steps, in declaration order (gravestones as kind "removed"; names only the rows hold as declared: false): wait_timeout (a wait_for: step's wait_timeout: in seconds, "dynamic" when a lambda or a method name sets it per run, null without one), counts (runs sitting at the step now by run status ready/performing/waiting/paused), run_ms (p50/p95/p99 of finished_at - started_at over completed or failed attempts) and pickup_ms (p50/p95 of scheduled → started), rates (failed: an error was recorded, reattempted, skipped) and outcomes over the sample_size attempts that finished in the window; percentiles over fewer than 20 samples are noisy. Also: started/finished/cancelled, e2e_ms, duration_histogram (finished runs per created → finished range), throughput (runs started/finished/cancelled per bucket_seconds), funnel (of the runs started in the window: reached = runs with a row for the step, stopped = where each run is now by status) and by_revision (the five app revisions that finished an attempt last, newest first, totals and per step).
15
15
  DESC
16
16
 
17
17
  input_schema(
@@ -9,7 +9,7 @@ module Pgbus
9
9
  tool_name "pgbus_workflows_attention"
10
10
  title "Pgbus Workflows Needing Attention"
11
11
  description <<~DESC
12
- Ongoing Pgbus::Workflow runs that need a person, grouped by (workflow_class, step_name, reason, error_class): reason is paused_error, stalled, paused_manual, overdue (due but not picked up past workflow_stall_threshold) or waiting_overdue (waiting past workflow_waiting_visibility_threshold without a wait_timeout). Each group has its run count, first_seen_at/last_seen_at, the status its runs are in and the actions that status allows; total_runs sums the groups (at most 200 groups, biggest first). workflow_class narrows them to one class. The newest error message is redacted unless the server allows payloads and include_payloads is set.
12
+ Ongoing Pgbus::Workflow runs that need a person, grouped by (workflow_class, step_name, reason, error_class): reason is paused_error, stalled, paused_manual, overdue (due but not picked up past workflow_stall_threshold) or waiting_overdue (waiting past workflow_waiting_visibility_threshold without a deadline: no wait_timeout, or a per-run one that set none or cleared it after a :notify notice). Each group has its run count, first_seen_at/last_seen_at, the status its runs are in and the actions that status allows; total_runs sums the groups (at most 200 groups, biggest first). workflow_class narrows them to one class. The newest error message is redacted unless the server allows payloads and include_payloads is set.
13
13
  DESC
14
14
 
15
15
  input_schema(
data/lib/pgbus/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Pgbus
4
- VERSION = "0.20.1"
4
+ VERSION = "0.21.1"
5
5
  end
@@ -158,12 +158,14 @@ module Pgbus
158
158
  end
159
159
  end
160
160
 
161
- # { step_id => the first deadline its wait missed } for waiting steps
162
- # with a deadline, one query: the first wait_timed_out event of each.
161
+ # { step_id => the first deadline its wait missed } for steps waiting
162
+ # on a signal, one query: the first wait_timed_out event of each.
163
163
  # Only on_timeout: :notify leaves a step waiting after that event
164
- # (issue #565): every other on_timeout ends the attempt.
164
+ # (issue #565): every other on_timeout ends the attempt. A per-run
165
+ # wait_timeout: may have cleared the deadline at that notice (issue
166
+ # #574), so a step without one is looked up too.
165
167
  def late_since(waiting)
166
- steps = waiting.select { |step| step_attribute(step, :timeout_at) }
168
+ steps = waiting.select { |step| step_attribute(step, :wait_for) }
167
169
  return {} if steps.empty?
168
170
 
169
171
  WorkflowEvent.where(workflow_id: steps.map(&:workflow_id), step_id: steps.map(&:id),
@@ -299,9 +299,9 @@ module Pgbus
299
299
  # duplicate (the original is still in the queue waiting for retry).
300
300
  # Falls back to enqueueing a fresh copy only if the original is gone
301
301
  # (e.g., already moved to DLQ).
302
- msg_id = event["msg_id"]
303
- if msg_id && @client.message_exists?(event["queue_name"], msg_id: msg_id.to_i)
304
- @client.set_visibility_timeout(event["queue_name"], msg_id.to_i, vt: 0)
302
+ physical = failed_message_queue(event)
303
+ if physical
304
+ @client.set_visibility_timeout(physical, event["msg_id"].to_i, vt: 0, prefixed: false)
305
305
  else
306
306
  payload = JSON.parse(event["payload"])
307
307
  headers = event["headers"]
@@ -1721,12 +1721,29 @@ module Pgbus
1721
1721
  nil
1722
1722
  end
1723
1723
 
1724
+ # The physical queue holding a failed job's message, or nil when it is
1725
+ # gone. The row records the logical queue only, and under priority
1726
+ # routing a msg_id repeats across the _pN sub-queues, so the job's
1727
+ # job_id must match too (issue #577). Two sub-queues' failures with the
1728
+ # same msg_id share one row, which keeps the first one's payload: that
1729
+ # is the message found (issue #583).
1730
+ def failed_message_queue(event)
1731
+ return unless event["queue_name"] && event["msg_id"]
1732
+
1733
+ @client.locate_message(event["queue_name"], msg_id: event["msg_id"].to_i, job_id: failed_job_id(event))
1734
+ end
1735
+
1736
+ def failed_job_id(event)
1737
+ payload = event["payload"]
1738
+ payload = JSON.parse(payload) if payload.is_a?(String)
1739
+ payload["job_id"] if payload.is_a?(Hash)
1740
+ end
1741
+
1724
1742
  # Archive the queue message a failed_event row points to. Idempotent —
1725
1743
  # silently no-ops if the message no longer exists in the queue.
1726
1744
  def archive_failed_message(event)
1727
- return unless event["queue_name"] && event["msg_id"]
1728
-
1729
- @client.archive_message(event["queue_name"], event["msg_id"].to_i)
1745
+ physical = failed_message_queue(event)
1746
+ @client.archive_message(physical, event["msg_id"].to_i, prefixed: false) if physical
1730
1747
  rescue StandardError => e
1731
1748
  Pgbus.logger.debug do
1732
1749
  "[Pgbus::Web] Error archiving message for failed event #{event["id"]}: #{e.message}"
@@ -1734,30 +1751,30 @@ module Pgbus
1734
1751
  end
1735
1752
 
1736
1753
  # Archive every queue message referenced by a failed_event row.
1737
- # Groups by queue and uses archive_batch so an N-event discard is
1738
- # one SQL statement per queue instead of N per-row roundtrips.
1739
- # Falls back to per-row archive inside a rescue if a batch fails,
1740
- # so one bad queue can't block progress on the others.
1754
+ # Groups by queue, then locates and archive_batches per physical table,
1755
+ # so an N-event discard is two SQL statements per table instead of N
1756
+ # per-row roundtrips. Falls back to per-row archive inside a rescue if
1757
+ # a batch fails, so one bad queue can't block progress on the others.
1741
1758
  def archive_all_failed_messages
1742
1759
  rows = connection.select_all(
1743
- "SELECT id, queue_name, msg_id FROM pgbus_failed_events WHERE msg_id IS NOT NULL AND #{NOT_HANDLER_QUEUE}",
1760
+ "SELECT id, queue_name, msg_id, payload FROM pgbus_failed_events WHERE msg_id IS NOT NULL AND #{NOT_HANDLER_QUEUE}",
1744
1761
  "Pgbus Collect Failed Messages", [handler_queue_names_literal]
1745
1762
  )
1746
1763
 
1747
1764
  grouped = rows.to_a.group_by { |row| row["queue_name"] }
1748
1765
  grouped.each do |queue_name, events|
1749
- msg_ids = events.filter_map { |row| row["msg_id"]&.to_i }
1750
- next if msg_ids.empty?
1751
-
1752
- begin
1753
- @client.archive_batch(queue_name, msg_ids)
1754
- rescue StandardError => e
1755
- Pgbus.logger.debug do
1756
- "[Pgbus::Web] archive_batch failed for #{queue_name} (#{e.message}); " \
1757
- "falling back to per-row archive"
1758
- end
1759
- events.each { |row| archive_failed_message(row) }
1766
+ # A priority queue's msg_ids repeat across its _pN sub-queues, so
1767
+ # each message is located first and archived from its own table.
1768
+ located = @client.locate_messages(queue_name, events.map { |row| [row["msg_id"].to_i, failed_job_id(row)] })
1769
+ located.group_by { |_msg_id, physical| physical }.each do |physical, pairs|
1770
+ @client.archive_batch(physical, pairs.map(&:first), prefixed: false)
1771
+ end
1772
+ rescue StandardError => e
1773
+ Pgbus.logger.debug do
1774
+ "[Pgbus::Web] archive_batch failed for #{queue_name} (#{e.message}); " \
1775
+ "falling back to per-row archive"
1760
1776
  end
1777
+ events.each { |row| archive_failed_message(row) }
1761
1778
  end
1762
1779
  rescue StandardError => e
1763
1780
  Pgbus.logger.debug { "[Pgbus::Web] Error archiving failed messages: #{e.message}" }
@@ -47,14 +47,14 @@ module Pgbus
47
47
  )
48
48
  end
49
49
 
50
- # [{ name:, kind:, removed:, declared: }] in pipeline order.
50
+ # [{ name:, kind:, removed:, declared:, wait_timeout: }] in pipeline order.
51
51
  def step_list(klass, stats, ongoing, reached, stopped)
52
52
  seen = (stats.keys + ongoing.map { |row| row[:step] } + reached.keys + stopped.keys.map(&:first)).compact.uniq
53
53
  declared = klass ? klass.workflow_steps.entries.map { |entry| declared_step(entry) } : []
54
54
  extra = (seen - declared.map { |entry| entry[:name] }).sort_by do |name|
55
55
  [reached.dig(name, :position) || Float::INFINITY, stats.dig(name, :first_seen_at)&.to_f || Float::INFINITY, name]
56
56
  end
57
- declared + extra.map { |name| { name: name, kind: nil, removed: false, declared: false } }
57
+ declared + extra.map { |name| { name: name, kind: nil, removed: false, declared: false, wait_timeout: nil } }
58
58
  end
59
59
 
60
60
  def declared_step(entry)
@@ -63,7 +63,15 @@ module Pgbus
63
63
  elsif entry.waits_for_signal? then "signal"
64
64
  else "step"
65
65
  end
66
- { name: entry.name, kind: kind, removed: entry.removed?, declared: true }
66
+ { name: entry.name, kind: kind, removed: entry.removed?, declared: true, wait_timeout: wait_timeout(entry) }
67
+ end
68
+
69
+ # A signal step's wait_timeout: in seconds, "dynamic" for a per-run one
70
+ # (issue #574), nil without one.
71
+ def wait_timeout(entry)
72
+ return nil if entry.removed?
73
+
74
+ entry.dynamic_wait_timeout? ? "dynamic" : entry.wait_timeout
67
75
  end
68
76
 
69
77
  def step(entry, stats, ongoing)
@@ -51,9 +51,12 @@ module Pgbus
51
51
  return ["waiting", "waiting_step", { step: step }] unless row[:waiting_for]
52
52
 
53
53
  args = { signal: row[:waiting_for], ago: row[:waiting_since] }
54
- return ["waiting", "waiting_signal", args] unless row[:timeout_at]
54
+ return signal_deadline(row, args.merge(time: row[:timeout_at])) if row[:timeout_at]
55
+ # A per-run wait_timeout: reported once under on_timeout: :notify,
56
+ # then left without a deadline (issue #574).
57
+ return ["waiting", "waiting_signal_late_no_deadline", args.merge(late: row[:late_since])] if row[:late_since]
55
58
 
56
- signal_deadline(row, args.merge(time: row[:timeout_at]))
59
+ ["waiting", "waiting_signal", args]
57
60
  end
58
61
 
59
62
  # An on_timeout: :notify deadline sends a late notice and the wait goes
@@ -94,6 +94,33 @@ module Pgbus
94
94
  @timed_out == true
95
95
  end
96
96
 
97
+ # Inside a step (its guards and its body): true when this attempt holds
98
+ # a signal before its gate runs: signal! woke the step, or an earlier
99
+ # attempt of it took one and reattempted or failed into a pause. False
100
+ # on an attempt that has not waited yet (even one whose gate then takes
101
+ # a buffered signal), after an on_timeout: :run wake, and in a step
102
+ # without wait_for:. The same answer for the whole attempt.
103
+ def woken?
104
+ step_context!(:woken?)
105
+ @woken = !@signal_peek&.held.nil? if @woken.nil?
106
+ @woken
107
+ end
108
+
109
+ # Inside a step (its guards and its body): true when the gate runs this
110
+ # attempt with a signal: the one it holds (woken?), or a pending one its
111
+ # wait_for: matcher takes. The guard for "skip a wait the record already
112
+ # passed", which neither skips a signalled step nor leaves its signal
113
+ # pending. False after an on_timeout: :run wake (the gate takes no
114
+ # signal then) and in a step without wait_for:. Reads only: the gate
115
+ # claims the signal after the guards.
116
+ #
117
+ # step :await_shipped, skip_if: -> { subject.shipped? && !signal_ready? }, wait_for: :shipped do … end
118
+ def signal_ready?
119
+ step_context!(:signal_ready?)
120
+ @signal_ready = !@signal_peek&.ready.nil? if @signal_ready.nil?
121
+ @signal_ready
122
+ end
123
+
97
124
  # Deliver a signal to this run. Stored under the run's lock; a step
98
125
  # waiting for it (wait_for:) is woken at once, otherwise it waits,
99
126
  # pending, for the step that claims it. With an idempotency_key a
@@ -164,19 +191,25 @@ module Pgbus
164
191
  @guarding = false
165
192
  end
166
193
 
167
- # Used by the StepJob only.
168
- def enter_step(context)
194
+ # Used by the StepJob only. +signals+ is the attempt's SignalPeek, which
195
+ # woken? and signal_ready? read until the gate answers them.
196
+ def enter_step(context, signals: nil)
169
197
  @step_context = context
198
+ @signal_peek = signals
170
199
  @output_values = nil
171
200
  @pending_output = nil
172
201
  @received_signal = nil
173
202
  @timed_out = false
203
+ @woken = nil
204
+ @signal_ready = nil
174
205
  end
175
206
 
176
207
  # Used by the StepJob's signal gate only.
177
- def enter_wait(signal:, timed_out:)
208
+ def enter_wait(signal:, timed_out:, woken: false)
178
209
  @received_signal = signal
179
210
  @timed_out = timed_out
211
+ @woken = woken
212
+ @signal_ready = !signal.nil?
180
213
  end
181
214
  end
182
215
  end
@@ -68,11 +68,11 @@ module Pgbus
68
68
 
69
69
  # --- start! and rerun! --------------------------------------------
70
70
 
71
- def start(klass, subject:, arguments:, idempotency_key: nil)
71
+ def start(klass, subject:, arguments:, idempotency_key: nil, actor: DEFAULT_ACTOR, reason: nil)
72
72
  attrs = run_attributes(klass, idempotency_key: idempotency_key)
73
73
  .merge(Payload.subject_attributes(subject))
74
74
  .merge(arguments: Payload.dump(arguments, what: "arguments of #{klass.name}"))
75
- insert_run(klass, attrs, actor: DEFAULT_ACTOR)
75
+ insert_run(klass, attrs, actor: actor, reason: reason)
76
76
  end
77
77
 
78
78
  # A finished or cancelled run started again: same subject and
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ class Workflow
5
+ # What a wait_for: step's attempt holds and what its gate would run it
6
+ # with (issue #572), read without claiming anything. The StepJob builds
7
+ # one per attempt: the step readers woken? and signal_ready? read it in
8
+ # the guards, and the gate decides with the same one, so the gate takes
9
+ # exactly the signal signal_ready? promised. All of it runs under the
10
+ # run's row lock, so each answer is looked up (and a matcher run) once.
11
+ class SignalPeek
12
+ def initialize(record, step, definition, now: Time.current)
13
+ @record = record
14
+ @step = step
15
+ @definition = definition
16
+ @now = now
17
+ end
18
+
19
+ # The claimed signal the step holds (signal! woke it, or an earlier
20
+ # attempt took one and reattempted or paused), or nil. Always nil for a
21
+ # step without wait_for:.
22
+ def held
23
+ return @held if defined?(@held)
24
+
25
+ @held = waits? ? WorkflowSignal.held_by_step(@record.workflow_id, @step.step_name) : nil
26
+ end
27
+
28
+ # The sweep woke this attempt with on_timeout: :run: its body runs
29
+ # without a signal (Signals.timed_out?).
30
+ def timed_out? = Signals.timed_out?(@step, @now)
31
+
32
+ # The signal the gate runs this attempt with: the one the step holds,
33
+ # else the oldest pending one its matcher takes. nil when the wait timed
34
+ # out (the body runs without one), when nothing matches (the step
35
+ # parks), and for a step without wait_for:.
36
+ def ready
37
+ return @ready if defined?(@ready)
38
+
39
+ @ready = waits? && !timed_out? ? held || Signals.match(@record, @definition.wait_for) : nil
40
+ end
41
+
42
+ private
43
+
44
+ def waits? = @definition.waits_for_signal?
45
+ end
46
+ end
47
+ end
@@ -12,6 +12,8 @@ module Pgbus
12
12
  # lock: a signal this step already holds (woken by signal!, or claimed by
13
13
  # an earlier attempt that reattempted or paused on a failure), else the
14
14
  # oldest pending one its matcher takes, else the step parks as waiting.
15
+ # The guards run first; woken? and signal_ready? (issue #572) show them
16
+ # what the gate holds and would take, through SignalPeek, claiming nothing.
15
17
  #
16
18
  # A claimed signal is consumed when its step ends, except when that
17
19
  # attempt reattempts or fails into a pause: the next attempt runs with it.
@@ -38,23 +40,38 @@ module Pgbus
38
40
  end
39
41
 
40
42
  # The gate of a wait_for: step. True when the body runs (with a signal,
41
- # or timed out), false when the step parked. Sets received_signal and
42
- # timed_out? on +workflow+.
43
- def gate(record, step, definition, workflow, now: Time.current)
44
- if timed_out?(step, now)
43
+ # or timed out), false when the step parked. Sets received_signal,
44
+ # timed_out?, woken? and signal_ready? on +workflow+. Which signal it
45
+ # takes is SignalPeek#ready: given the attempt's +peek+ (the StepJob
46
+ # passes the one its guards read), the very answer signal_ready? gave.
47
+ def gate(record, step, definition, workflow, now: Time.current, peek: nil)
48
+ peek ||= SignalPeek.new(record, step, definition, now: now)
49
+ if peek.timed_out?
45
50
  workflow.send(:enter_wait, signal: nil, timed_out: true)
46
51
  return true
47
52
  end
48
53
 
49
- signal = WorkflowSignal.held_by_step(record.workflow_id, step.step_name) || match(record, definition.wait_for)
50
- return park(record, step, definition, now) unless signal
54
+ signal = peek.ready
55
+ return park(record, step, definition, workflow, now) unless signal
51
56
 
57
+ woken = !peek.held.nil?
52
58
  claim!(record, step, signal, actor: Runner::SYSTEM_ACTOR) if signal.status == "pending"
53
59
  step.update_columns(signal_id: signal.id) unless step.signal_id == signal.id
54
- workflow.send(:enter_wait, signal: signal, timed_out: false)
60
+ workflow.send(:enter_wait, signal: signal, timed_out: false, woken: woken)
55
61
  true
56
62
  end
57
63
 
64
+ # An attempt that ended before its gate (cancel_if, skip_if, a guard
65
+ # that raised, a removed_step) records +held+, the signal the step
66
+ # holds, on its row, so settle treats it as for any attempt: consumed
67
+ # when the step ends, kept when it reattempts or fails into a pause. A
68
+ # post-reattempt row carries no signal_id until its gate runs. A
69
+ # pending signal stays pending: no step took it, and a later wait_for:
70
+ # step may.
71
+ def adopt_held(step, held)
72
+ step.update_columns(signal_id: held.id) if held && step.signal_id != held.id
73
+ end
74
+
58
75
  # The step's attempt ended (StepOutcome.apply): consume the signal it
59
76
  # holds, unless the next attempt runs with it.
60
77
  def settle(step, status, column, now:)
@@ -116,14 +133,18 @@ module Pgbus
116
133
  nil
117
134
  end
118
135
 
119
- def park(record, step, definition, now) # rubocop:disable Naming/PredicateMethod -- the gate's false: the body does not run
120
- timeout_at = definition.wait_timeout && (now + definition.wait_timeout)
136
+ # A per-run wait_timeout: is evaluated here, before Waiting.park! writes
137
+ # the waiting state: one that raises (an invalid return raises
138
+ # ArgumentError) leaves the attempt in_progress for the step's exception
139
+ # policies; nil parks it without a deadline.
140
+ def park(record, step, definition, workflow, now) # rubocop:disable Naming/PredicateMethod -- the gate's false: the body does not run
141
+ timeout_at = definition.timeout_at_for(workflow, now)
121
142
  Waiting.park!(record, step, now: now, wait_for: definition.wait_for_name, timeout_at: timeout_at)
122
143
  false
123
144
  end
124
145
 
125
146
  # This attempt waited, holds no signal, and its wait_timeout passed: the
126
- # sweep woke it with on_timeout: :run.
147
+ # sweep woke it with on_timeout: :run (SignalPeek#timed_out?).
127
148
  def timed_out?(step, now)
128
149
  !step.waiting_since.nil? && step.signal_id.nil? && !step.timeout_at.nil? && step.timeout_at <= now + 1
129
150
  end