pgbus 0.20.1 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +66 -8
- data/README.md +1 -1
- data/app/helpers/pgbus/workflow_insights_helper.rb +8 -0
- data/app/models/pgbus/workflow_entry.rb +3 -2
- data/app/views/pgbus/workflows/definitions/_step_card.html.erb +3 -0
- data/config/locales/da.yml +3 -0
- data/config/locales/de.yml +3 -0
- data/config/locales/en.yml +3 -0
- data/config/locales/es.yml +3 -0
- data/config/locales/fi.yml +3 -0
- data/config/locales/fr.yml +3 -0
- data/config/locales/it.yml +3 -0
- data/config/locales/ja.yml +3 -0
- data/config/locales/nb.yml +3 -0
- data/config/locales/nl.yml +3 -0
- data/config/locales/pt.yml +3 -0
- data/config/locales/sv.yml +3 -0
- data/lib/generators/pgbus/templates/add_workflows.rb.erb +3 -1
- data/lib/generators/pgbus/templates/migration.rb.erb +3 -1
- data/lib/pgbus/batch/sweep.rb +10 -5
- data/lib/pgbus/client.rb +109 -42
- data/lib/pgbus/configuration/workflow_settings.rb +1 -1
- data/lib/pgbus/mcp/tools/workflow_stats_tool.rb +1 -1
- data/lib/pgbus/mcp/tools/workflows_attention_tool.rb +1 -1
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source/workflow_list.rb +6 -4
- data/lib/pgbus/web/data_source.rb +39 -22
- data/lib/pgbus/web/workflow_insights_report.rb +11 -3
- data/lib/pgbus/web/workflow_state.rb +5 -2
- data/lib/pgbus/workflow/instance.rb +36 -3
- data/lib/pgbus/workflow/runner.rb +2 -2
- data/lib/pgbus/workflow/signal_peek.rb +47 -0
- data/lib/pgbus/workflow/signals.rb +31 -10
- data/lib/pgbus/workflow/step_definition.rb +45 -3
- data/lib/pgbus/workflow/step_execution.rb +27 -10
- data/lib/pgbus/workflow/wait_timeout.rb +38 -7
- data/lib/pgbus/workflow.rb +11 -5
- data/sig/pgbus/client.rbs +7 -5
- data/sig/pgbus.rbs +43 -0
- metadata +2 -1
data/lib/pgbus/client.rb
CHANGED
|
@@ -720,16 +720,24 @@ module Pgbus
|
|
|
720
720
|
# reaper to determine if a uniqueness lock with msg_id=0 (placeholder)
|
|
721
721
|
# still has a corresponding queue message.
|
|
722
722
|
#
|
|
723
|
-
# +queue_name+
|
|
724
|
-
#
|
|
723
|
+
# +queue_name+ is a logical name (e.g. "default"), expanded through the
|
|
724
|
+
# queue strategy, unless the caller passes +prefixed: false+ for a physical
|
|
725
|
+
# table name (e.g. "pgbus_default_p1"), used as is. The name's spelling
|
|
726
|
+
# decides nothing: a logical "pgbus_mailers" lives in pgbus_pgbus_mailers
|
|
727
|
+
# (issue #569).
|
|
725
728
|
#
|
|
726
729
|
# Returns:
|
|
727
730
|
# true — the message definitely exists in the queue
|
|
728
731
|
# false — the message definitely does not exist
|
|
729
732
|
# nil — could not determine (e.g. queue table missing or unknown error).
|
|
730
733
|
# Callers MUST treat nil as "exists" for safety.
|
|
731
|
-
|
|
732
|
-
|
|
734
|
+
#
|
|
735
|
+
# Under priority routing a logical name probes every `_pN` table, and each
|
|
736
|
+
# table has its own msg_id sequence: a bare +msg_id+ may match a different
|
|
737
|
+
# message in a sibling sub-queue. A caller that knows the sub-queue passes
|
|
738
|
+
# its physical name with +prefixed: false+.
|
|
739
|
+
def message_in_queue?(queue_name, msg_id:, prefixed: true)
|
|
740
|
+
message_exists?(queue_name, msg_id: msg_id, prefixed: prefixed)
|
|
733
741
|
end
|
|
734
742
|
|
|
735
743
|
# Look up a queue row by ActiveJob job_id in the JSON payload. Used by the
|
|
@@ -738,10 +746,10 @@ module Pgbus
|
|
|
738
746
|
#
|
|
739
747
|
# Same tri-state as message_exists?: true / false / nil (unknown). A
|
|
740
748
|
# logical name expands to every physical table the strategy owns (the
|
|
741
|
-
# `_pN` sub-queues under priority routing);
|
|
742
|
-
#
|
|
743
|
-
def message_with_job_id?(queue_name, job_id:)
|
|
744
|
-
tables = lookup_physical_queue_names(queue_name).map { |name| QueueNameValidator.sanitize!(name) }
|
|
749
|
+
# `_pN` sub-queues under priority routing); a physical name passed with
|
|
750
|
+
# +prefixed: false+ probes itself only.
|
|
751
|
+
def message_with_job_id?(queue_name, job_id:, prefixed: true)
|
|
752
|
+
tables = lookup_physical_queue_names(queue_name, prefixed: prefixed).map { |name| QueueNameValidator.sanitize!(name) }
|
|
745
753
|
determined = false
|
|
746
754
|
|
|
747
755
|
synchronized do
|
|
@@ -758,43 +766,85 @@ module Pgbus
|
|
|
758
766
|
determined ? false : nil # rubocop:disable Style/ReturnNilInPredicateMethodDefinition -- tri-state: nil means unknown
|
|
759
767
|
end
|
|
760
768
|
|
|
761
|
-
#
|
|
762
|
-
#
|
|
763
|
-
def
|
|
764
|
-
|
|
769
|
+
# The physical table that holds message +msg_id+, or nil when none does.
|
|
770
|
+
# See locate_messages.
|
|
771
|
+
def locate_message(queue_name, msg_id:, job_id: nil, prefixed: true)
|
|
772
|
+
locate_messages(queue_name, [[msg_id, job_id]], prefixed: prefixed)[msg_id.to_i]
|
|
773
|
+
end
|
|
774
|
+
|
|
775
|
+
# { msg_id => physical table } for each [msg_id, job_id] pair whose
|
|
776
|
+
# message is queued, one query per table on one connection. A logical
|
|
777
|
+
# name expands to every table the strategy owns, and under priority
|
|
778
|
+
# routing each `_pN` sub-queue numbers its messages from its own
|
|
779
|
+
# sequence: only a message carrying the pair's job_id matches, or any
|
|
780
|
+
# message with that msg_id when job_id is nil, the first table winning
|
|
781
|
+
# (issue #577). A physical name with +prefixed: false+ is probed alone.
|
|
782
|
+
# A table that does not exist holds nothing; other errors raise.
|
|
783
|
+
def locate_messages(queue_name, messages, prefixed: true)
|
|
784
|
+
wanted = messages.to_h { |msg_id, job_id| [msg_id.to_i, job_id&.to_s] }
|
|
785
|
+
tables = lookup_physical_queue_names(queue_name, prefixed: prefixed)
|
|
786
|
+
located = {}
|
|
787
|
+
|
|
788
|
+
synchronized do
|
|
789
|
+
with_raw_connection do |conn|
|
|
790
|
+
tables.each do |name|
|
|
791
|
+
pending = wanted.keys - located.keys
|
|
792
|
+
break if pending.empty?
|
|
793
|
+
|
|
794
|
+
queued_job_ids(conn, QueueNameValidator.sanitize!(name), pending).each do |row|
|
|
795
|
+
msg_id = row["msg_id"].to_i
|
|
796
|
+
job_id = wanted[msg_id]
|
|
797
|
+
located[msg_id] ||= name if job_id.nil? || row["job_id"] == job_id
|
|
798
|
+
end
|
|
799
|
+
end
|
|
800
|
+
end
|
|
801
|
+
end
|
|
802
|
+
|
|
803
|
+
located
|
|
804
|
+
end
|
|
805
|
+
|
|
806
|
+
# DLQ companion of a logical queue name, or of a physical table name passed
|
|
807
|
+
# with +prefixed: false+.
|
|
808
|
+
def dead_letter_physical_name(queue_name, prefixed: true)
|
|
809
|
+
return config.dead_letter_queue_name(queue_name) if prefixed
|
|
810
|
+
|
|
765
811
|
# Priority sub-queues share the logical queue's DLQ (`pgbus_default_dlq`),
|
|
766
|
-
# not a per-priority `…_pN_dlq`. Only
|
|
767
|
-
#
|
|
768
|
-
|
|
812
|
+
# not a per-priority `…_pN_dlq`. Only a physical name can be a sub-queue,
|
|
813
|
+
# and only when the strategy actually creates `_pN` tables: a logical
|
|
814
|
+
# queue named `orders_p0` keeps its suffix.
|
|
815
|
+
name = queue_name.to_s
|
|
816
|
+
base = @queue_strategy.priority? ? name.sub(/_p\d+\z/, "") : name
|
|
769
817
|
"#{base}#{Pgbus::DEAD_LETTER_SUFFIX}"
|
|
770
818
|
end
|
|
771
819
|
|
|
772
820
|
# Same tri-state as message_exists?: true / false / nil (unknown).
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
821
|
+
# A logical name probes the archive of every physical table the strategy
|
|
822
|
+
# owns, like message_exists?; a physical name (prefixed: false) its own.
|
|
823
|
+
# A bare msg_id is unique per table only (see message_in_queue?).
|
|
824
|
+
def message_archived?(queue_name, msg_id:, prefixed: true)
|
|
825
|
+
tables = lookup_physical_queue_names(queue_name, prefixed: prefixed).map { |name| QueueNameValidator.sanitize!(name) }
|
|
826
|
+
determined = false
|
|
776
827
|
|
|
777
828
|
synchronized do
|
|
778
829
|
with_raw_connection do |conn|
|
|
779
|
-
|
|
830
|
+
tables.each do |sanitized|
|
|
831
|
+
archived = probe_archive_presence(conn, sanitized, msg_id.to_i)
|
|
832
|
+
return true if archived == true
|
|
833
|
+
|
|
834
|
+
determined = true unless archived == :missing
|
|
835
|
+
end
|
|
780
836
|
end
|
|
781
837
|
end
|
|
782
|
-
rescue ActiveRecord::StatementInvalid => e
|
|
783
|
-
raise unless undefined_table_error?(e)
|
|
784
|
-
|
|
785
|
-
nil
|
|
786
|
-
rescue StandardError => e
|
|
787
|
-
raise unless defined?(PG::UndefinedTable) && e.is_a?(PG::UndefinedTable)
|
|
788
838
|
|
|
789
|
-
nil
|
|
839
|
+
determined ? false : nil # rubocop:disable Style/ReturnNilInPredicateMethodDefinition -- tri-state: nil means unknown
|
|
790
840
|
end
|
|
791
841
|
|
|
792
|
-
def message_exists?(queue_name, msg_id: nil, uniqueness_key: nil)
|
|
842
|
+
def message_exists?(queue_name, msg_id: nil, uniqueness_key: nil, prefixed: true)
|
|
793
843
|
has_msg_id = !msg_id.nil?
|
|
794
844
|
has_uniqueness_key = !uniqueness_key.nil?
|
|
795
845
|
raise ArgumentError, "pass msg_id, uniqueness_key, or both" unless has_msg_id || has_uniqueness_key
|
|
796
846
|
|
|
797
|
-
tables = lookup_physical_queue_names(queue_name).map { |name| QueueNameValidator.sanitize!(name) }
|
|
847
|
+
tables = lookup_physical_queue_names(queue_name, prefixed: prefixed).map { |name| QueueNameValidator.sanitize!(name) }
|
|
798
848
|
determined = false
|
|
799
849
|
|
|
800
850
|
synchronized do
|
|
@@ -1092,21 +1142,10 @@ module Pgbus
|
|
|
1092
1142
|
end
|
|
1093
1143
|
end
|
|
1094
1144
|
|
|
1095
|
-
# Accept either a logical name ("default") or an already-prefixed
|
|
1096
|
-
# physical name ("pgbus_default") and return the physical name.
|
|
1097
|
-
# Coerces symbols to strings so callers can pass either form.
|
|
1098
|
-
def resolve_full_queue_name(queue_name)
|
|
1099
|
-
name = queue_name.to_s
|
|
1100
|
-
prefix = "#{config.queue_prefix}_"
|
|
1101
|
-
name.start_with?(prefix) ? name : config.queue_name(name)
|
|
1102
|
-
end
|
|
1103
|
-
|
|
1104
1145
|
# Logical names expand through QueueFactory so a priority queue's _pN
|
|
1105
|
-
# tables are included;
|
|
1106
|
-
def lookup_physical_queue_names(queue_name)
|
|
1107
|
-
|
|
1108
|
-
prefix = "#{config.queue_prefix}_"
|
|
1109
|
-
name.start_with?(prefix) ? [name] : @queue_strategy.physical_queue_names(name)
|
|
1146
|
+
# tables are included; a physical name (prefixed: false) stays as is.
|
|
1147
|
+
def lookup_physical_queue_names(queue_name, prefixed:)
|
|
1148
|
+
prefixed ? @queue_strategy.physical_queue_names(queue_name.to_s) : [queue_name.to_s]
|
|
1110
1149
|
end
|
|
1111
1150
|
|
|
1112
1151
|
def probe_queue_presence(conn, sanitized, msg_id:, uniqueness_key:)
|
|
@@ -1125,6 +1164,18 @@ module Pgbus
|
|
|
1125
1164
|
:missing
|
|
1126
1165
|
end
|
|
1127
1166
|
|
|
1167
|
+
def probe_archive_presence(conn, sanitized, msg_id)
|
|
1168
|
+
msg_id_archived?(conn, sanitized, msg_id)
|
|
1169
|
+
rescue ActiveRecord::StatementInvalid => e
|
|
1170
|
+
raise unless undefined_table_error?(e)
|
|
1171
|
+
|
|
1172
|
+
:missing
|
|
1173
|
+
rescue StandardError => e
|
|
1174
|
+
raise unless defined?(PG::UndefinedTable) && e.is_a?(PG::UndefinedTable)
|
|
1175
|
+
|
|
1176
|
+
:missing
|
|
1177
|
+
end
|
|
1178
|
+
|
|
1128
1179
|
def probe_job_id_presence(conn, sanitized, job_id)
|
|
1129
1180
|
job_id_present?(conn, sanitized, job_id)
|
|
1130
1181
|
rescue ActiveRecord::StatementInvalid => e
|
|
@@ -1137,6 +1188,22 @@ module Pgbus
|
|
|
1137
1188
|
:missing
|
|
1138
1189
|
end
|
|
1139
1190
|
|
|
1191
|
+
# msg_ids are integers, so the array literal is safe.
|
|
1192
|
+
def queued_job_ids(conn, sanitized, msg_ids)
|
|
1193
|
+
conn.exec_params(
|
|
1194
|
+
"SELECT msg_id, message::jsonb ->> 'job_id' AS job_id FROM pgmq.q_#{sanitized} WHERE msg_id = ANY($1::bigint[])",
|
|
1195
|
+
["{#{msg_ids.join(",")}}"]
|
|
1196
|
+
)
|
|
1197
|
+
rescue ActiveRecord::StatementInvalid => e
|
|
1198
|
+
raise unless undefined_table_error?(e)
|
|
1199
|
+
|
|
1200
|
+
[]
|
|
1201
|
+
rescue StandardError => e
|
|
1202
|
+
raise unless defined?(PG::UndefinedTable) && e.is_a?(PG::UndefinedTable)
|
|
1203
|
+
|
|
1204
|
+
[]
|
|
1205
|
+
end
|
|
1206
|
+
|
|
1140
1207
|
def scan_uniqueness_keys(conn, sanitized, keys, found)
|
|
1141
1208
|
placeholders = keys.each_index.map { |i| "$#{i + 1}" }.join(", ")
|
|
1142
1209
|
result = conn.exec_params(
|
|
@@ -12,7 +12,7 @@ module Pgbus
|
|
|
12
12
|
# config.workflow_max_payload_size = 131_072 # bytes: arguments and step outputs
|
|
13
13
|
# config.workflow_error_backtrace_lines = 50 # backtrace lines kept on a failed step
|
|
14
14
|
# config.workflow_waiting_visibility_threshold = 7.days # a step waiting this long without a
|
|
15
|
-
# #
|
|
15
|
+
# # deadline counts as stuck (waiting_overdue)
|
|
16
16
|
# config.workflow_subject_url = ->(type, id) { "/admin/#{type.underscore.pluralize}/#{id}" }
|
|
17
17
|
# # the dashboard's subject link (nil: plain text)
|
|
18
18
|
module WorkflowSettings
|
|
@@ -11,7 +11,7 @@ module Pgbus
|
|
|
11
11
|
tool_name "pgbus_workflow_stats"
|
|
12
12
|
title "Pgbus Workflow Stats"
|
|
13
13
|
description <<~DESC
|
|
14
|
-
One Pgbus::Workflow class over a window (1h, 24h default, 7d or 30d, capped at workflow_retention). steps, in declaration order (gravestones as kind "removed"; names only the rows hold as declared: false): counts (runs sitting at the step now by run status ready/performing/waiting/paused), run_ms (p50/p95/p99 of finished_at - started_at over completed or failed attempts) and pickup_ms (p50/p95 of scheduled → started), rates (failed: an error was recorded, reattempted, skipped) and outcomes over the sample_size attempts that finished in the window; percentiles over fewer than 20 samples are noisy. Also: started/finished/cancelled, e2e_ms, duration_histogram (finished runs per created → finished range), throughput (runs started/finished/cancelled per bucket_seconds), funnel (of the runs started in the window: reached = runs with a row for the step, stopped = where each run is now by status) and by_revision (the five app revisions that finished an attempt last, newest first, totals and per step).
|
|
14
|
+
One Pgbus::Workflow class over a window (1h, 24h default, 7d or 30d, capped at workflow_retention). steps, in declaration order (gravestones as kind "removed"; names only the rows hold as declared: false): wait_timeout (a wait_for: step's wait_timeout: in seconds, "dynamic" when a lambda or a method name sets it per run, null without one), counts (runs sitting at the step now by run status ready/performing/waiting/paused), run_ms (p50/p95/p99 of finished_at - started_at over completed or failed attempts) and pickup_ms (p50/p95 of scheduled → started), rates (failed: an error was recorded, reattempted, skipped) and outcomes over the sample_size attempts that finished in the window; percentiles over fewer than 20 samples are noisy. Also: started/finished/cancelled, e2e_ms, duration_histogram (finished runs per created → finished range), throughput (runs started/finished/cancelled per bucket_seconds), funnel (of the runs started in the window: reached = runs with a row for the step, stopped = where each run is now by status) and by_revision (the five app revisions that finished an attempt last, newest first, totals and per step).
|
|
15
15
|
DESC
|
|
16
16
|
|
|
17
17
|
input_schema(
|
|
@@ -9,7 +9,7 @@ module Pgbus
|
|
|
9
9
|
tool_name "pgbus_workflows_attention"
|
|
10
10
|
title "Pgbus Workflows Needing Attention"
|
|
11
11
|
description <<~DESC
|
|
12
|
-
Ongoing Pgbus::Workflow runs that need a person, grouped by (workflow_class, step_name, reason, error_class): reason is paused_error, stalled, paused_manual, overdue (due but not picked up past workflow_stall_threshold) or waiting_overdue (waiting past workflow_waiting_visibility_threshold without a wait_timeout). Each group has its run count, first_seen_at/last_seen_at, the status its runs are in and the actions that status allows; total_runs sums the groups (at most 200 groups, biggest first). workflow_class narrows them to one class. The newest error message is redacted unless the server allows payloads and include_payloads is set.
|
|
12
|
+
Ongoing Pgbus::Workflow runs that need a person, grouped by (workflow_class, step_name, reason, error_class): reason is paused_error, stalled, paused_manual, overdue (due but not picked up past workflow_stall_threshold) or waiting_overdue (waiting past workflow_waiting_visibility_threshold without a deadline: no wait_timeout, or a per-run one that set none or cleared it after a :notify notice). Each group has its run count, first_seen_at/last_seen_at, the status its runs are in and the actions that status allows; total_runs sums the groups (at most 200 groups, biggest first). workflow_class narrows them to one class. The newest error message is redacted unless the server allows payloads and include_payloads is set.
|
|
13
13
|
DESC
|
|
14
14
|
|
|
15
15
|
input_schema(
|
data/lib/pgbus/version.rb
CHANGED
|
@@ -158,12 +158,14 @@ module Pgbus
|
|
|
158
158
|
end
|
|
159
159
|
end
|
|
160
160
|
|
|
161
|
-
# { step_id => the first deadline its wait missed } for waiting
|
|
162
|
-
#
|
|
161
|
+
# { step_id => the first deadline its wait missed } for steps waiting
|
|
162
|
+
# on a signal, one query: the first wait_timed_out event of each.
|
|
163
163
|
# Only on_timeout: :notify leaves a step waiting after that event
|
|
164
|
-
# (issue #565): every other on_timeout ends the attempt.
|
|
164
|
+
# (issue #565): every other on_timeout ends the attempt. A per-run
|
|
165
|
+
# wait_timeout: may have cleared the deadline at that notice (issue
|
|
166
|
+
# #574), so a step without one is looked up too.
|
|
165
167
|
def late_since(waiting)
|
|
166
|
-
steps = waiting.select { |step| step_attribute(step, :
|
|
168
|
+
steps = waiting.select { |step| step_attribute(step, :wait_for) }
|
|
167
169
|
return {} if steps.empty?
|
|
168
170
|
|
|
169
171
|
WorkflowEvent.where(workflow_id: steps.map(&:workflow_id), step_id: steps.map(&:id),
|
|
@@ -299,9 +299,9 @@ module Pgbus
|
|
|
299
299
|
# duplicate (the original is still in the queue waiting for retry).
|
|
300
300
|
# Falls back to enqueueing a fresh copy only if the original is gone
|
|
301
301
|
# (e.g., already moved to DLQ).
|
|
302
|
-
|
|
303
|
-
if
|
|
304
|
-
@client.set_visibility_timeout(event["
|
|
302
|
+
physical = failed_message_queue(event)
|
|
303
|
+
if physical
|
|
304
|
+
@client.set_visibility_timeout(physical, event["msg_id"].to_i, vt: 0, prefixed: false)
|
|
305
305
|
else
|
|
306
306
|
payload = JSON.parse(event["payload"])
|
|
307
307
|
headers = event["headers"]
|
|
@@ -1721,12 +1721,29 @@ module Pgbus
|
|
|
1721
1721
|
nil
|
|
1722
1722
|
end
|
|
1723
1723
|
|
|
1724
|
+
# The physical queue holding a failed job's message, or nil when it is
|
|
1725
|
+
# gone. The row records the logical queue only, and under priority
|
|
1726
|
+
# routing a msg_id repeats across the _pN sub-queues, so the job's
|
|
1727
|
+
# job_id must match too (issue #577). Two sub-queues' failures with the
|
|
1728
|
+
# same msg_id share one row, which keeps the first one's payload: that
|
|
1729
|
+
# is the message found (issue #583).
|
|
1730
|
+
def failed_message_queue(event)
|
|
1731
|
+
return unless event["queue_name"] && event["msg_id"]
|
|
1732
|
+
|
|
1733
|
+
@client.locate_message(event["queue_name"], msg_id: event["msg_id"].to_i, job_id: failed_job_id(event))
|
|
1734
|
+
end
|
|
1735
|
+
|
|
1736
|
+
def failed_job_id(event)
|
|
1737
|
+
payload = event["payload"]
|
|
1738
|
+
payload = JSON.parse(payload) if payload.is_a?(String)
|
|
1739
|
+
payload["job_id"] if payload.is_a?(Hash)
|
|
1740
|
+
end
|
|
1741
|
+
|
|
1724
1742
|
# Archive the queue message a failed_event row points to. Idempotent —
|
|
1725
1743
|
# silently no-ops if the message no longer exists in the queue.
|
|
1726
1744
|
def archive_failed_message(event)
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
@client.archive_message(event["queue_name"], event["msg_id"].to_i)
|
|
1745
|
+
physical = failed_message_queue(event)
|
|
1746
|
+
@client.archive_message(physical, event["msg_id"].to_i, prefixed: false) if physical
|
|
1730
1747
|
rescue StandardError => e
|
|
1731
1748
|
Pgbus.logger.debug do
|
|
1732
1749
|
"[Pgbus::Web] Error archiving message for failed event #{event["id"]}: #{e.message}"
|
|
@@ -1734,30 +1751,30 @@ module Pgbus
|
|
|
1734
1751
|
end
|
|
1735
1752
|
|
|
1736
1753
|
# Archive every queue message referenced by a failed_event row.
|
|
1737
|
-
# Groups by queue
|
|
1738
|
-
#
|
|
1739
|
-
# Falls back to per-row archive inside a rescue if
|
|
1740
|
-
# so one bad queue can't block progress on the others.
|
|
1754
|
+
# Groups by queue, then locates and archive_batches per physical table,
|
|
1755
|
+
# so an N-event discard is two SQL statements per table instead of N
|
|
1756
|
+
# per-row roundtrips. Falls back to per-row archive inside a rescue if
|
|
1757
|
+
# a batch fails, so one bad queue can't block progress on the others.
|
|
1741
1758
|
def archive_all_failed_messages
|
|
1742
1759
|
rows = connection.select_all(
|
|
1743
|
-
"SELECT id, queue_name, msg_id FROM pgbus_failed_events WHERE msg_id IS NOT NULL AND #{NOT_HANDLER_QUEUE}",
|
|
1760
|
+
"SELECT id, queue_name, msg_id, payload FROM pgbus_failed_events WHERE msg_id IS NOT NULL AND #{NOT_HANDLER_QUEUE}",
|
|
1744
1761
|
"Pgbus Collect Failed Messages", [handler_queue_names_literal]
|
|
1745
1762
|
)
|
|
1746
1763
|
|
|
1747
1764
|
grouped = rows.to_a.group_by { |row| row["queue_name"] }
|
|
1748
1765
|
grouped.each do |queue_name, events|
|
|
1749
|
-
msg_ids
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
@client.archive_batch(
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
events.each { |row| archive_failed_message(row) }
|
|
1766
|
+
# A priority queue's msg_ids repeat across its _pN sub-queues, so
|
|
1767
|
+
# each message is located first and archived from its own table.
|
|
1768
|
+
located = @client.locate_messages(queue_name, events.map { |row| [row["msg_id"].to_i, failed_job_id(row)] })
|
|
1769
|
+
located.group_by { |_msg_id, physical| physical }.each do |physical, pairs|
|
|
1770
|
+
@client.archive_batch(physical, pairs.map(&:first), prefixed: false)
|
|
1771
|
+
end
|
|
1772
|
+
rescue StandardError => e
|
|
1773
|
+
Pgbus.logger.debug do
|
|
1774
|
+
"[Pgbus::Web] archive_batch failed for #{queue_name} (#{e.message}); " \
|
|
1775
|
+
"falling back to per-row archive"
|
|
1760
1776
|
end
|
|
1777
|
+
events.each { |row| archive_failed_message(row) }
|
|
1761
1778
|
end
|
|
1762
1779
|
rescue StandardError => e
|
|
1763
1780
|
Pgbus.logger.debug { "[Pgbus::Web] Error archiving failed messages: #{e.message}" }
|
|
@@ -47,14 +47,14 @@ module Pgbus
|
|
|
47
47
|
)
|
|
48
48
|
end
|
|
49
49
|
|
|
50
|
-
# [{ name:, kind:, removed:, declared: }] in pipeline order.
|
|
50
|
+
# [{ name:, kind:, removed:, declared:, wait_timeout: }] in pipeline order.
|
|
51
51
|
def step_list(klass, stats, ongoing, reached, stopped)
|
|
52
52
|
seen = (stats.keys + ongoing.map { |row| row[:step] } + reached.keys + stopped.keys.map(&:first)).compact.uniq
|
|
53
53
|
declared = klass ? klass.workflow_steps.entries.map { |entry| declared_step(entry) } : []
|
|
54
54
|
extra = (seen - declared.map { |entry| entry[:name] }).sort_by do |name|
|
|
55
55
|
[reached.dig(name, :position) || Float::INFINITY, stats.dig(name, :first_seen_at)&.to_f || Float::INFINITY, name]
|
|
56
56
|
end
|
|
57
|
-
declared + extra.map { |name| { name: name, kind: nil, removed: false, declared: false } }
|
|
57
|
+
declared + extra.map { |name| { name: name, kind: nil, removed: false, declared: false, wait_timeout: nil } }
|
|
58
58
|
end
|
|
59
59
|
|
|
60
60
|
def declared_step(entry)
|
|
@@ -63,7 +63,15 @@ module Pgbus
|
|
|
63
63
|
elsif entry.waits_for_signal? then "signal"
|
|
64
64
|
else "step"
|
|
65
65
|
end
|
|
66
|
-
{ name: entry.name, kind: kind, removed: entry.removed?, declared: true }
|
|
66
|
+
{ name: entry.name, kind: kind, removed: entry.removed?, declared: true, wait_timeout: wait_timeout(entry) }
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# A signal step's wait_timeout: in seconds, "dynamic" for a per-run one
|
|
70
|
+
# (issue #574), nil without one.
|
|
71
|
+
def wait_timeout(entry)
|
|
72
|
+
return nil if entry.removed?
|
|
73
|
+
|
|
74
|
+
entry.dynamic_wait_timeout? ? "dynamic" : entry.wait_timeout
|
|
67
75
|
end
|
|
68
76
|
|
|
69
77
|
def step(entry, stats, ongoing)
|
|
@@ -51,9 +51,12 @@ module Pgbus
|
|
|
51
51
|
return ["waiting", "waiting_step", { step: step }] unless row[:waiting_for]
|
|
52
52
|
|
|
53
53
|
args = { signal: row[:waiting_for], ago: row[:waiting_since] }
|
|
54
|
-
return
|
|
54
|
+
return signal_deadline(row, args.merge(time: row[:timeout_at])) if row[:timeout_at]
|
|
55
|
+
# A per-run wait_timeout: reported once under on_timeout: :notify,
|
|
56
|
+
# then left without a deadline (issue #574).
|
|
57
|
+
return ["waiting", "waiting_signal_late_no_deadline", args.merge(late: row[:late_since])] if row[:late_since]
|
|
55
58
|
|
|
56
|
-
|
|
59
|
+
["waiting", "waiting_signal", args]
|
|
57
60
|
end
|
|
58
61
|
|
|
59
62
|
# An on_timeout: :notify deadline sends a late notice and the wait goes
|
|
@@ -94,6 +94,33 @@ module Pgbus
|
|
|
94
94
|
@timed_out == true
|
|
95
95
|
end
|
|
96
96
|
|
|
97
|
+
# Inside a step (its guards and its body): true when this attempt holds
|
|
98
|
+
# a signal before its gate runs: signal! woke the step, or an earlier
|
|
99
|
+
# attempt of it took one and reattempted or failed into a pause. False
|
|
100
|
+
# on an attempt that has not waited yet (even one whose gate then takes
|
|
101
|
+
# a buffered signal), after an on_timeout: :run wake, and in a step
|
|
102
|
+
# without wait_for:. The same answer for the whole attempt.
|
|
103
|
+
def woken?
|
|
104
|
+
step_context!(:woken?)
|
|
105
|
+
@woken = !@signal_peek&.held.nil? if @woken.nil?
|
|
106
|
+
@woken
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# Inside a step (its guards and its body): true when the gate runs this
|
|
110
|
+
# attempt with a signal: the one it holds (woken?), or a pending one its
|
|
111
|
+
# wait_for: matcher takes. The guard for "skip a wait the record already
|
|
112
|
+
# passed", which neither skips a signalled step nor leaves its signal
|
|
113
|
+
# pending. False after an on_timeout: :run wake (the gate takes no
|
|
114
|
+
# signal then) and in a step without wait_for:. Reads only: the gate
|
|
115
|
+
# claims the signal after the guards.
|
|
116
|
+
#
|
|
117
|
+
# step :await_shipped, skip_if: -> { subject.shipped? && !signal_ready? }, wait_for: :shipped do … end
|
|
118
|
+
def signal_ready?
|
|
119
|
+
step_context!(:signal_ready?)
|
|
120
|
+
@signal_ready = !@signal_peek&.ready.nil? if @signal_ready.nil?
|
|
121
|
+
@signal_ready
|
|
122
|
+
end
|
|
123
|
+
|
|
97
124
|
# Deliver a signal to this run. Stored under the run's lock; a step
|
|
98
125
|
# waiting for it (wait_for:) is woken at once, otherwise it waits,
|
|
99
126
|
# pending, for the step that claims it. With an idempotency_key a
|
|
@@ -164,19 +191,25 @@ module Pgbus
|
|
|
164
191
|
@guarding = false
|
|
165
192
|
end
|
|
166
193
|
|
|
167
|
-
# Used by the StepJob only.
|
|
168
|
-
|
|
194
|
+
# Used by the StepJob only. +signals+ is the attempt's SignalPeek, which
|
|
195
|
+
# woken? and signal_ready? read until the gate answers them.
|
|
196
|
+
def enter_step(context, signals: nil)
|
|
169
197
|
@step_context = context
|
|
198
|
+
@signal_peek = signals
|
|
170
199
|
@output_values = nil
|
|
171
200
|
@pending_output = nil
|
|
172
201
|
@received_signal = nil
|
|
173
202
|
@timed_out = false
|
|
203
|
+
@woken = nil
|
|
204
|
+
@signal_ready = nil
|
|
174
205
|
end
|
|
175
206
|
|
|
176
207
|
# Used by the StepJob's signal gate only.
|
|
177
|
-
def enter_wait(signal:, timed_out:)
|
|
208
|
+
def enter_wait(signal:, timed_out:, woken: false)
|
|
178
209
|
@received_signal = signal
|
|
179
210
|
@timed_out = timed_out
|
|
211
|
+
@woken = woken
|
|
212
|
+
@signal_ready = !signal.nil?
|
|
180
213
|
end
|
|
181
214
|
end
|
|
182
215
|
end
|
|
@@ -68,11 +68,11 @@ module Pgbus
|
|
|
68
68
|
|
|
69
69
|
# --- start! and rerun! --------------------------------------------
|
|
70
70
|
|
|
71
|
-
def start(klass, subject:, arguments:, idempotency_key: nil)
|
|
71
|
+
def start(klass, subject:, arguments:, idempotency_key: nil, actor: DEFAULT_ACTOR, reason: nil)
|
|
72
72
|
attrs = run_attributes(klass, idempotency_key: idempotency_key)
|
|
73
73
|
.merge(Payload.subject_attributes(subject))
|
|
74
74
|
.merge(arguments: Payload.dump(arguments, what: "arguments of #{klass.name}"))
|
|
75
|
-
insert_run(klass, attrs, actor:
|
|
75
|
+
insert_run(klass, attrs, actor: actor, reason: reason)
|
|
76
76
|
end
|
|
77
77
|
|
|
78
78
|
# A finished or cancelled run started again: same subject and
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class Workflow
|
|
5
|
+
# What a wait_for: step's attempt holds and what its gate would run it
|
|
6
|
+
# with (issue #572), read without claiming anything. The StepJob builds
|
|
7
|
+
# one per attempt: the step readers woken? and signal_ready? read it in
|
|
8
|
+
# the guards, and the gate decides with the same one, so the gate takes
|
|
9
|
+
# exactly the signal signal_ready? promised. All of it runs under the
|
|
10
|
+
# run's row lock, so each answer is looked up (and a matcher run) once.
|
|
11
|
+
class SignalPeek
|
|
12
|
+
def initialize(record, step, definition, now: Time.current)
|
|
13
|
+
@record = record
|
|
14
|
+
@step = step
|
|
15
|
+
@definition = definition
|
|
16
|
+
@now = now
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# The claimed signal the step holds (signal! woke it, or an earlier
|
|
20
|
+
# attempt took one and reattempted or paused), or nil. Always nil for a
|
|
21
|
+
# step without wait_for:.
|
|
22
|
+
def held
|
|
23
|
+
return @held if defined?(@held)
|
|
24
|
+
|
|
25
|
+
@held = waits? ? WorkflowSignal.held_by_step(@record.workflow_id, @step.step_name) : nil
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# The sweep woke this attempt with on_timeout: :run: its body runs
|
|
29
|
+
# without a signal (Signals.timed_out?).
|
|
30
|
+
def timed_out? = Signals.timed_out?(@step, @now)
|
|
31
|
+
|
|
32
|
+
# The signal the gate runs this attempt with: the one the step holds,
|
|
33
|
+
# else the oldest pending one its matcher takes. nil when the wait timed
|
|
34
|
+
# out (the body runs without one), when nothing matches (the step
|
|
35
|
+
# parks), and for a step without wait_for:.
|
|
36
|
+
def ready
|
|
37
|
+
return @ready if defined?(@ready)
|
|
38
|
+
|
|
39
|
+
@ready = waits? && !timed_out? ? held || Signals.match(@record, @definition.wait_for) : nil
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
def waits? = @definition.waits_for_signal?
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -12,6 +12,8 @@ module Pgbus
|
|
|
12
12
|
# lock: a signal this step already holds (woken by signal!, or claimed by
|
|
13
13
|
# an earlier attempt that reattempted or paused on a failure), else the
|
|
14
14
|
# oldest pending one its matcher takes, else the step parks as waiting.
|
|
15
|
+
# The guards run first; woken? and signal_ready? (issue #572) show them
|
|
16
|
+
# what the gate holds and would take, through SignalPeek, claiming nothing.
|
|
15
17
|
#
|
|
16
18
|
# A claimed signal is consumed when its step ends, except when that
|
|
17
19
|
# attempt reattempts or fails into a pause: the next attempt runs with it.
|
|
@@ -38,23 +40,38 @@ module Pgbus
|
|
|
38
40
|
end
|
|
39
41
|
|
|
40
42
|
# The gate of a wait_for: step. True when the body runs (with a signal,
|
|
41
|
-
# or timed out), false when the step parked. Sets received_signal
|
|
42
|
-
# timed_out? on +workflow+.
|
|
43
|
-
|
|
44
|
-
|
|
43
|
+
# or timed out), false when the step parked. Sets received_signal,
|
|
44
|
+
# timed_out?, woken? and signal_ready? on +workflow+. Which signal it
|
|
45
|
+
# takes is SignalPeek#ready: given the attempt's +peek+ (the StepJob
|
|
46
|
+
# passes the one its guards read), the very answer signal_ready? gave.
|
|
47
|
+
def gate(record, step, definition, workflow, now: Time.current, peek: nil)
|
|
48
|
+
peek ||= SignalPeek.new(record, step, definition, now: now)
|
|
49
|
+
if peek.timed_out?
|
|
45
50
|
workflow.send(:enter_wait, signal: nil, timed_out: true)
|
|
46
51
|
return true
|
|
47
52
|
end
|
|
48
53
|
|
|
49
|
-
signal =
|
|
50
|
-
return park(record, step, definition, now) unless signal
|
|
54
|
+
signal = peek.ready
|
|
55
|
+
return park(record, step, definition, workflow, now) unless signal
|
|
51
56
|
|
|
57
|
+
woken = !peek.held.nil?
|
|
52
58
|
claim!(record, step, signal, actor: Runner::SYSTEM_ACTOR) if signal.status == "pending"
|
|
53
59
|
step.update_columns(signal_id: signal.id) unless step.signal_id == signal.id
|
|
54
|
-
workflow.send(:enter_wait, signal: signal, timed_out: false)
|
|
60
|
+
workflow.send(:enter_wait, signal: signal, timed_out: false, woken: woken)
|
|
55
61
|
true
|
|
56
62
|
end
|
|
57
63
|
|
|
64
|
+
# An attempt that ended before its gate (cancel_if, skip_if, a guard
|
|
65
|
+
# that raised, a removed_step) records +held+, the signal the step
|
|
66
|
+
# holds, on its row, so settle treats it as for any attempt: consumed
|
|
67
|
+
# when the step ends, kept when it reattempts or fails into a pause. A
|
|
68
|
+
# post-reattempt row carries no signal_id until its gate runs. A
|
|
69
|
+
# pending signal stays pending: no step took it, and a later wait_for:
|
|
70
|
+
# step may.
|
|
71
|
+
def adopt_held(step, held)
|
|
72
|
+
step.update_columns(signal_id: held.id) if held && step.signal_id != held.id
|
|
73
|
+
end
|
|
74
|
+
|
|
58
75
|
# The step's attempt ended (StepOutcome.apply): consume the signal it
|
|
59
76
|
# holds, unless the next attempt runs with it.
|
|
60
77
|
def settle(step, status, column, now:)
|
|
@@ -116,14 +133,18 @@ module Pgbus
|
|
|
116
133
|
nil
|
|
117
134
|
end
|
|
118
135
|
|
|
119
|
-
|
|
120
|
-
|
|
136
|
+
# A per-run wait_timeout: is evaluated here, before Waiting.park! writes
|
|
137
|
+
# the waiting state: one that raises (an invalid return raises
|
|
138
|
+
# ArgumentError) leaves the attempt in_progress for the step's exception
|
|
139
|
+
# policies; nil parks it without a deadline.
|
|
140
|
+
def park(record, step, definition, workflow, now) # rubocop:disable Naming/PredicateMethod -- the gate's false: the body does not run
|
|
141
|
+
timeout_at = definition.timeout_at_for(workflow, now)
|
|
121
142
|
Waiting.park!(record, step, now: now, wait_for: definition.wait_for_name, timeout_at: timeout_at)
|
|
122
143
|
false
|
|
123
144
|
end
|
|
124
145
|
|
|
125
146
|
# This attempt waited, holds no signal, and its wait_timeout passed: the
|
|
126
|
-
# sweep woke it with on_timeout: :run.
|
|
147
|
+
# sweep woke it with on_timeout: :run (SignalPeek#timed_out?).
|
|
127
148
|
def timed_out?(step, now)
|
|
128
149
|
!step.waiting_since.nil? && step.signal_id.nil? && !step.timeout_at.nil? && step.timeout_at <= now + 1
|
|
129
150
|
end
|