pgbus 0.19.4 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/README.md +97 -7
- data/Rakefile +6 -1
- data/app/controllers/pgbus/dashboard_controller.rb +2 -0
- data/app/controllers/pgbus/workflow_definitions_controller.rb +28 -0
- data/app/controllers/pgbus/workflows_controller.rb +197 -0
- data/app/frontend/pgbus/modules/charts.js +148 -4
- data/app/frontend/pgbus/modules/charts.min.js +1 -1
- data/app/frontend/pgbus/modules/charts.min.js.map +1 -1
- data/app/frontend/pgbus/style.css +1 -1
- data/app/helpers/pgbus/button_helper.rb +4 -2
- data/app/helpers/pgbus/workflow_insights_helper.rb +129 -0
- data/app/helpers/pgbus/workflows_helper.rb +165 -0
- data/app/models/pgbus/blocked_execution.rb +12 -0
- data/app/models/pgbus/uniqueness_key.rb +88 -3
- data/app/models/pgbus/workflow_entry.rb +172 -0
- data/app/models/pgbus/workflow_event.rb +43 -0
- data/app/models/pgbus/workflow_signal.rb +77 -0
- data/app/models/pgbus/workflow_step.rb +149 -0
- data/app/views/layouts/pgbus/application.html.erb +2 -0
- data/app/views/pgbus/dashboard/_stats_cards.html.erb +17 -0
- data/app/views/pgbus/workflows/_actions.html.erb +59 -0
- data/app/views/pgbus/workflows/_events_table.html.erb +24 -0
- data/app/views/pgbus/workflows/_filters.html.erb +29 -0
- data/app/views/pgbus/workflows/_signal_form.html.erb +31 -0
- data/app/views/pgbus/workflows/_signals_table.html.erb +40 -0
- data/app/views/pgbus/workflows/_status_filter.html.erb +12 -0
- data/app/views/pgbus/workflows/_steps_table.html.erb +79 -0
- data/app/views/pgbus/workflows/_timeline.html.erb +49 -0
- data/app/views/pgbus/workflows/_workflows_table.html.erb +59 -0
- data/app/views/pgbus/workflows/attention.html.erb +88 -0
- data/app/views/pgbus/workflows/definitions/_by_revision.html.erb +52 -0
- data/app/views/pgbus/workflows/definitions/_duration_table.html.erb +18 -0
- data/app/views/pgbus/workflows/definitions/_funnel.html.erb +49 -0
- data/app/views/pgbus/workflows/definitions/_pipeline.html.erb +16 -0
- data/app/views/pgbus/workflows/definitions/_step_card.html.erb +31 -0
- data/app/views/pgbus/workflows/definitions/_throughput_table.html.erb +32 -0
- data/app/views/pgbus/workflows/definitions/_window.html.erb +17 -0
- data/app/views/pgbus/workflows/definitions/index.html.erb +71 -0
- data/app/views/pgbus/workflows/definitions/not_found.html.erb +5 -0
- data/app/views/pgbus/workflows/definitions/show.html.erb +80 -0
- data/app/views/pgbus/workflows/index.html.erb +15 -0
- data/app/views/pgbus/workflows/not_found.html.erb +5 -0
- data/app/views/pgbus/workflows/show.html.erb +93 -0
- data/config/i18n-tasks.yml +6 -0
- data/config/locales/da.yml +445 -0
- data/config/locales/de.yml +445 -0
- data/config/locales/en.yml +445 -0
- data/config/locales/es.yml +445 -0
- data/config/locales/fi.yml +445 -0
- data/config/locales/fr.yml +445 -0
- data/config/locales/it.yml +445 -0
- data/config/locales/ja.yml +445 -0
- data/config/locales/nb.yml +445 -0
- data/config/locales/nl.yml +445 -0
- data/config/locales/pt.yml +445 -0
- data/config/locales/sv.yml +445 -0
- data/config/routes.rb +26 -0
- data/lib/generators/pgbus/add_workflow_signals_generator.rb +56 -0
- data/lib/generators/pgbus/add_workflows_generator.rb +54 -0
- data/lib/generators/pgbus/templates/add_workflow_signals.rb.erb +51 -0
- data/lib/generators/pgbus/templates/add_workflows.rb.erb +150 -0
- data/lib/generators/pgbus/templates/migration.rb.erb +127 -0
- data/lib/pgbus/active_job/adapter.rb +81 -3
- data/lib/pgbus/active_job/delivery.rb +63 -0
- data/lib/pgbus/active_job/executor.rb +20 -8
- data/lib/pgbus/concurrency/blocked_execution.rb +14 -6
- data/lib/pgbus/configuration/workflow_settings.rb +99 -0
- data/lib/pgbus/configuration.rb +64 -4
- data/lib/pgbus/dead_letter_header.rb +9 -1
- data/lib/pgbus/generators/migration_detector.rb +40 -0
- data/lib/pgbus/instrumentation.rb +33 -0
- data/lib/pgbus/integrations/appsignal/dashboard.json +157 -1
- data/lib/pgbus/integrations/appsignal/subscriber.rb +16 -1
- data/lib/pgbus/mcp/rack_app.rb +7 -2
- data/lib/pgbus/mcp/runner.rb +2 -1
- data/lib/pgbus/mcp/serialization.rb +25 -0
- data/lib/pgbus/mcp/server.rb +53 -10
- data/lib/pgbus/mcp/tools/deploy_tasks_tool.rb +1 -10
- data/lib/pgbus/mcp/tools/workflow_cancel_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_definitions_tool.rb +36 -0
- data/lib/pgbus/mcp/tools/workflow_detail_tool.rb +38 -0
- data/lib/pgbus/mcp/tools/workflow_resume_at_tool.rb +32 -0
- data/lib/pgbus/mcp/tools/workflow_resume_tool.rb +24 -0
- data/lib/pgbus/mcp/tools/workflow_run_now_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_signal_tool.rb +77 -0
- data/lib/pgbus/mcp/tools/workflow_skip_step_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_stats_tool.rb +42 -0
- data/lib/pgbus/mcp/tools/workflows_attention_tool.rb +42 -0
- data/lib/pgbus/mcp/tools/workflows_tool.rb +60 -0
- data/lib/pgbus/mcp/workflow_insights_payload.rb +15 -0
- data/lib/pgbus/mcp/workflow_payloads.rb +53 -0
- data/lib/pgbus/mcp/write_tool.rb +134 -0
- data/lib/pgbus/mcp.rb +15 -0
- data/lib/pgbus/metrics/backends/statsd.rb +6 -0
- data/lib/pgbus/metrics/subscriber.rb +14 -1
- data/lib/pgbus/metrics/workflow_metrics.rb +75 -0
- data/lib/pgbus/process/consumer.rb +2 -1
- data/lib/pgbus/process/dispatcher.rb +39 -3
- data/lib/pgbus/process/worker.rb +10 -1
- data/lib/pgbus/recurring/schedule.rb +1 -1
- data/lib/pgbus/streams/presence.rb +65 -36
- data/lib/pgbus/uniqueness.rb +23 -6
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source/workflow_actions.rb +109 -0
- data/lib/pgbus/web/data_source/workflow_insights.rb +99 -0
- data/lib/pgbus/web/data_source/workflow_list.rb +291 -0
- data/lib/pgbus/web/data_source.rb +126 -15
- data/lib/pgbus/web/workflow_action_plan.rb +62 -0
- data/lib/pgbus/web/workflow_insights_queries.rb +250 -0
- data/lib/pgbus/web/workflow_insights_report.rb +171 -0
- data/lib/pgbus/web/workflow_payload.rb +36 -0
- data/lib/pgbus/web/workflow_query.rb +82 -0
- data/lib/pgbus/web/workflow_state.rb +89 -0
- data/lib/pgbus/web/workflow_timeline.rb +130 -0
- data/lib/pgbus/web/workflow_window.rb +51 -0
- data/lib/pgbus/workflow/batch_finished_job.rb +21 -0
- data/lib/pgbus/workflow/dsl.rb +105 -0
- data/lib/pgbus/workflow/error_capture.rb +63 -0
- data/lib/pgbus/workflow/exception_policy.rb +170 -0
- data/lib/pgbus/workflow/fan_out.rb +157 -0
- data/lib/pgbus/workflow/fan_out_definition.rb +41 -0
- data/lib/pgbus/workflow/flow_control.rb +94 -0
- data/lib/pgbus/workflow/instance.rb +183 -0
- data/lib/pgbus/workflow/outside_verbs.rb +138 -0
- data/lib/pgbus/workflow/payload.rb +53 -0
- data/lib/pgbus/workflow/registry.rb +53 -0
- data/lib/pgbus/workflow/removed_step.rb +14 -0
- data/lib/pgbus/workflow/runner.rb +218 -0
- data/lib/pgbus/workflow/signal_matcher.rb +53 -0
- data/lib/pgbus/workflow/signals.rb +132 -0
- data/lib/pgbus/workflow/states.rb +119 -0
- data/lib/pgbus/workflow/step_context.rb +58 -0
- data/lib/pgbus/workflow/step_definition.rb +133 -0
- data/lib/pgbus/workflow/step_execution.rb +242 -0
- data/lib/pgbus/workflow/step_job.rb +28 -0
- data/lib/pgbus/workflow/step_outcome.rb +151 -0
- data/lib/pgbus/workflow/steps.rb +103 -0
- data/lib/pgbus/workflow/sweep.rb +156 -0
- data/lib/pgbus/workflow/test_helpers.rb +152 -0
- data/lib/pgbus/workflow/wait_timeout.rb +94 -0
- data/lib/pgbus/workflow/waiting.rb +82 -0
- data/lib/pgbus/workflow.rb +183 -0
- data/sig/pgbus/configuration.rbs +23 -0
- metadata +92 -1
data/lib/pgbus/uniqueness.rb
CHANGED
|
@@ -16,8 +16,14 @@ module Pgbus
|
|
|
16
16
|
# or executing.
|
|
17
17
|
# 2. Execution: PGMQ's visibility timeout is the execution lock —
|
|
18
18
|
# no separate claim_for_execution step needed.
|
|
19
|
-
# 3.
|
|
20
|
-
#
|
|
19
|
+
# 3. Retry: a retry_job / retry_on re-enqueue passes through the key the
|
|
20
|
+
# running attempt holds, and the row is handed to the retry: unbound
|
|
21
|
+
# and re-stamped before the retry is sent or parked (unbind!), bound
|
|
22
|
+
# to it once sent (bind!).
|
|
23
|
+
# 4. Completion/DLQ: the finishing message deletes the row only while it
|
|
24
|
+
# holds it (release_held_by!), so a row handed to a retry, or a
|
|
25
|
+
# successor's, stays. The dashboard's discards do the same.
|
|
26
|
+
# 5. Crash recovery: if a worker dies, VT expires, the message becomes
|
|
21
27
|
# readable again. The uniqueness key row stays (correctly — the job
|
|
22
28
|
# hasn't finished). The next worker picks it up and executes.
|
|
23
29
|
#
|
|
@@ -179,14 +185,25 @@ module Pgbus
|
|
|
179
185
|
return :no_lock unless config[:strategy] == :until_executed
|
|
180
186
|
|
|
181
187
|
acquired = if msg_id && queue_name
|
|
182
|
-
UniquenessKey.acquire!(key, queue_name: queue_name, msg_id: msg_id)
|
|
188
|
+
UniquenessKey.acquire!(key, queue_name: lock_queue_name(queue_name), msg_id: msg_id)
|
|
183
189
|
else
|
|
184
190
|
# Pre-produce check: use advisory lock + ON CONFLICT
|
|
185
|
-
UniquenessKey.acquire!(key, queue_name: queue_name || PLACEHOLDER_QUEUE,
|
|
191
|
+
UniquenessKey.acquire!(key, queue_name: lock_queue_name(queue_name || PLACEHOLDER_QUEUE),
|
|
192
|
+
msg_id: msg_id || 0)
|
|
186
193
|
end
|
|
187
194
|
acquired ? :acquired : :locked
|
|
188
195
|
end
|
|
189
196
|
|
|
197
|
+
# The queue a job's uniqueness lock records: the logical name as its
|
|
198
|
+
# table spells it ("orders-handler" → "orders_handler"), the inverse of
|
|
199
|
+
# stripping the prefix off the table name. Every writer and releaser of
|
|
200
|
+
# a job's lock goes through it, so a release that knows only the table
|
|
201
|
+
# (the dashboard) or a worker that spells the queue the table's way
|
|
202
|
+
# still finds the row it was bound under (issue #564).
|
|
203
|
+
def lock_queue_name(queue_name, config: Pgbus.configuration)
|
|
204
|
+
config.queue_name(queue_name).delete_prefix("#{config.queue_prefix}_")
|
|
205
|
+
end
|
|
206
|
+
|
|
190
207
|
# Acquire the uniqueness lock at execution time (:while_executing only).
|
|
191
208
|
# Returns true if acquired, false if another instance is running.
|
|
192
209
|
#
|
|
@@ -199,7 +216,7 @@ module Pgbus
|
|
|
199
216
|
return true unless strategy == :while_executing
|
|
200
217
|
|
|
201
218
|
queue_name ||= payload["queue_name"] || "unknown"
|
|
202
|
-
UniquenessKey.acquire!(key, queue_name: queue_name, msg_id: msg_id.to_i,
|
|
219
|
+
UniquenessKey.acquire!(key, queue_name: lock_queue_name(queue_name), msg_id: msg_id.to_i,
|
|
203
220
|
reacquire_same_message: msg_id.to_i.positive?)
|
|
204
221
|
end
|
|
205
222
|
|
|
@@ -217,7 +234,7 @@ module Pgbus
|
|
|
217
234
|
return unless key
|
|
218
235
|
return unless msg_id.to_i.positive?
|
|
219
236
|
|
|
220
|
-
UniquenessKey.bind!(key, queue_name: queue_name, msg_id: msg_id)
|
|
237
|
+
UniquenessKey.bind!(key, queue_name: lock_queue_name(queue_name), msg_id: msg_id)
|
|
221
238
|
end
|
|
222
239
|
|
|
223
240
|
# True when the uniqueness row is not yet bound to a real PGMQ message.
|
data/lib/pgbus/version.rb
CHANGED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Web
|
|
5
|
+
class DataSource
|
|
6
|
+
# The Workflows dashboard's actions (issue #542). Each goes through the
|
|
7
|
+
# run's own outside verb (Pgbus::Workflow#resume! and friends), so the
|
|
8
|
+
# States table and the row lock decide, and returns the real outcome:
|
|
9
|
+
#
|
|
10
|
+
# { ok: true } it happened
|
|
11
|
+
# { ok: false, code:, reason:, status: } it did not, and why
|
|
12
|
+
#
|
|
13
|
+
# code is :not_allowed (the run is no longer in a state that allows it,
|
|
14
|
+
# usually a lost race: status says where it is now), :not_found,
|
|
15
|
+
# :invalid (an unknown step, a payload over the size cap) or :failed
|
|
16
|
+
# (anything else: logged). actor is who it is recorded as on the run's
|
|
17
|
+
# audit trail; reason is the operator's note.
|
|
18
|
+
module WorkflowActions
|
|
19
|
+
# At most this many runs per bulk call; a bigger group takes another press.
|
|
20
|
+
BULK_CAP = 500
|
|
21
|
+
# The outcomes a bulk call counts as "had already moved on".
|
|
22
|
+
MOVED_ON = %i[not_allowed not_found].freeze
|
|
23
|
+
|
|
24
|
+
def resume_workflow(workflow_id, actor:, reason: nil)
|
|
25
|
+
workflow_action(workflow_id) { |wf| wf.resume!(actor: actor, reason: reason) }
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def cancel_workflow(workflow_id, actor:, reason: nil)
|
|
29
|
+
workflow_action(workflow_id) { |wf| wf.cancel!(actor: actor, reason: reason) }
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def skip_workflow_step(workflow_id, actor:, reason: nil)
|
|
33
|
+
workflow_action(workflow_id) { |wf| wf.skip!(actor: actor, reason: reason) }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def resume_workflow_at(workflow_id, step:, actor:, reason: nil)
|
|
37
|
+
workflow_action(workflow_id) { |wf| wf.resume_at!(step, actor: actor, reason: reason) }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def run_workflow_step_now(workflow_id, actor:, reason: nil)
|
|
41
|
+
workflow_action(workflow_id) { |wf| wf.run_now!(actor: actor, reason: reason) }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# A redelivery with the same idempotency_key stores nothing: ok, with
|
|
45
|
+
# duplicate: true. The actor and reason are on the signal_received event.
|
|
46
|
+
def signal_workflow(workflow_id, name:, payload:, idempotency_key:, actor:, reason: nil)
|
|
47
|
+
duplicate = false
|
|
48
|
+
result = workflow_action(workflow_id) do |wf|
|
|
49
|
+
signal = wf.signal!(name, payload: payload, idempotency_key: idempotency_key, actor: actor, reason: reason)
|
|
50
|
+
duplicate = signal.duplicate_delivery?
|
|
51
|
+
end
|
|
52
|
+
result[:ok] ? result.merge(duplicate: duplicate) : result
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# The new run's workflow_id on success.
|
|
56
|
+
def rerun_workflow(workflow_id, actor:, reason: nil)
|
|
57
|
+
rerun = nil
|
|
58
|
+
result = workflow_action(workflow_id) { |wf| rerun = wf.rerun!(actor: actor, reason: reason) }
|
|
59
|
+
result[:ok] ? result.merge(workflow_id: rerun.workflow_id) : result
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# +group+ is a needs-attention group (workflow_class, step_name,
|
|
63
|
+
# reason, error_class). Each of its runs, at most BULK_CAP, through its
|
|
64
|
+
# own guarded transition: one that moved on in the meantime is
|
|
65
|
+
# skipped. { ok:, done:, skipped:, failed:, failure: }, or ok: false
|
|
66
|
+
# with a reason when the group could not be read.
|
|
67
|
+
def bulk_resume_workflows(group:, actor:, reason: nil)
|
|
68
|
+
bulk_workflow_action(group) { |id| resume_workflow(id, actor: actor, reason: reason) }
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def bulk_cancel_workflows(group:, actor:, reason: nil)
|
|
72
|
+
bulk_workflow_action(group) { |id| cancel_workflow(id, actor: actor, reason: reason) }
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
private
|
|
76
|
+
|
|
77
|
+
def workflow_action(workflow_id)
|
|
78
|
+
workflow = Workflow.find(workflow_id)
|
|
79
|
+
return { ok: false, code: :not_found, reason: "workflow #{workflow_id} not found" } unless workflow
|
|
80
|
+
|
|
81
|
+
yield workflow
|
|
82
|
+
{ ok: true }
|
|
83
|
+
rescue Workflow::InvalidTransition => e
|
|
84
|
+
{ ok: false, code: :not_allowed, reason: e.message, status: WorkflowEntry.where(workflow_id: workflow_id).pick(:status) }
|
|
85
|
+
rescue ActiveRecord::RecordNotFound => e
|
|
86
|
+
{ ok: false, code: :not_found, reason: e.message }
|
|
87
|
+
rescue Workflow::StepNotDefined, Workflow::PayloadTooLarge, ArgumentError => e
|
|
88
|
+
{ ok: false, code: :invalid, reason: e.message }
|
|
89
|
+
rescue StandardError => e
|
|
90
|
+
Pgbus.logger.error { "[Pgbus::Web] Workflow action on #{workflow_id} failed: #{e.class}: #{e.message}" }
|
|
91
|
+
{ ok: false, code: :failed, reason: "#{e.class}: #{e.message}" }
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# done: changed. skipped: had moved on (no longer in a state that
|
|
95
|
+
# allows it, or gone). failed: anything else, with the first reason.
|
|
96
|
+
def bulk_workflow_action(group, &)
|
|
97
|
+
ids = WorkflowEntry.attention_ids(group, **attention_thresholds, limit: BULK_CAP)
|
|
98
|
+
outcomes = ids.map(&)
|
|
99
|
+
moved_on, failed = outcomes.reject { |outcome| outcome[:ok] }.partition { |outcome| MOVED_ON.include?(outcome[:code]) }
|
|
100
|
+
result = { ok: true, done: outcomes.size - moved_on.size - failed.size, skipped: moved_on.size, failed: failed.size }
|
|
101
|
+
failed.empty? ? result : result.merge(failure: failed.first[:reason])
|
|
102
|
+
rescue StandardError => e
|
|
103
|
+
Pgbus.logger.error { "[Pgbus::Web] Bulk workflow action on #{group.inspect} failed: #{e.class}: #{e.message}" }
|
|
104
|
+
{ ok: false, code: :failed, reason: "#{e.class}: #{e.message}", done: 0, skipped: 0, failed: 0 }
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Web
|
|
5
|
+
class DataSource
|
|
6
|
+
# The workflow insights' reads (issue #549): one row per workflow class
|
|
7
|
+
# for /pgbus/workflows/definitions, and one class's pipeline, step
|
|
8
|
+
# performance, throughput, durations, funnel and per-revision numbers
|
|
9
|
+
# for /pgbus/workflows/definitions/:workflow_class and the MCP tools.
|
|
10
|
+
# Read from the step and run rows inside a WorkflowWindow
|
|
11
|
+
# (WorkflowInsightsQueries); WorkflowInsightsReport shapes the answer.
|
|
12
|
+
#
|
|
13
|
+
# A class is the registered Pgbus::Workflow subclass of that name, or,
|
|
14
|
+
# for a name present in pgbus_workflows, its safe_constantize when that
|
|
15
|
+
# is a Pgbus::Workflow subclass. A name from a URL that is neither is
|
|
16
|
+
# never constantized: the class is unknown.
|
|
17
|
+
module WorkflowInsights
|
|
18
|
+
# A workflow class name: a constant path (OrderWorkflow,
|
|
19
|
+
# Billing::OrderWorkflow). The route constrains to the same shape.
|
|
20
|
+
CLASS_NAME = /\A[A-Z][A-Za-z0-9_]*(?:::[A-Z][A-Za-z0-9_]*)*\z/
|
|
21
|
+
|
|
22
|
+
def workflow_definitions(window:)
|
|
23
|
+
return [] unless Workflow::Sweep.tables_ready?
|
|
24
|
+
|
|
25
|
+
ongoing = WorkflowInsightsQueries.ongoing.group_by { |row| row[:workflow_class] }
|
|
26
|
+
ended = WorkflowInsightsQueries.ended(since: window.since, until_time: window.now).index_by { |row| row[:workflow_class] }
|
|
27
|
+
stuck = attention_by_class
|
|
28
|
+
names = (registered_workflows.keys + WorkflowInsightsQueries.class_names + ongoing.keys + ended.keys + stuck.keys)
|
|
29
|
+
names.uniq.sort.map do |name|
|
|
30
|
+
WorkflowInsightsReport.summary(name, insights_class(name), ongoing.fetch(name, []), ended[name], stuck.fetch(name, 0))
|
|
31
|
+
end
|
|
32
|
+
rescue StandardError => e
|
|
33
|
+
Pgbus.logger.error { "[Pgbus::Web] Error reading workflow definitions: #{e.class}: #{e.message}" }
|
|
34
|
+
[]
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# nil for a name that is not a constant path, that no class or run
|
|
38
|
+
# has, or when the read failed.
|
|
39
|
+
def workflow_definition(workflow_class, window:)
|
|
40
|
+
return nil unless Workflow::Sweep.tables_ready?
|
|
41
|
+
|
|
42
|
+
name = workflow_class.to_s
|
|
43
|
+
return nil unless CLASS_NAME.match?(name)
|
|
44
|
+
|
|
45
|
+
registered = registered_workflows[name]
|
|
46
|
+
return nil unless registered || WorkflowInsightsQueries.class_known?(name)
|
|
47
|
+
|
|
48
|
+
read_definition(name, registered || insights_class(name), window)
|
|
49
|
+
rescue StandardError => e
|
|
50
|
+
Pgbus.logger.error { "[Pgbus::Web] Error reading workflow definition #{workflow_class}: #{e.class}: #{e.message}" }
|
|
51
|
+
nil
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
def read_definition(name, klass, window)
|
|
57
|
+
now = window.now
|
|
58
|
+
zone = Time.zone || ActiveSupport::TimeZone["UTC"]
|
|
59
|
+
since = window.since
|
|
60
|
+
queries = WorkflowInsightsQueries
|
|
61
|
+
WorkflowInsightsReport.detail(
|
|
62
|
+
name, klass,
|
|
63
|
+
window: window, since: since, now: now, ongoing: queries.ongoing(workflow_class: name),
|
|
64
|
+
ended: queries.ended(since: since, until_time: now, workflow_class: name).first,
|
|
65
|
+
attention: attention_by_class(workflow_class: name).fetch(name, 0),
|
|
66
|
+
stats: queries.step_stats(name, since: since, until_time: now),
|
|
67
|
+
histogram: queries.duration_histogram(name, since: since, until_time: now),
|
|
68
|
+
throughput: queries.throughput(name, since: since, until_time: now, bucket: window.bucket_seconds,
|
|
69
|
+
zone: zone.tzinfo.identifier),
|
|
70
|
+
zone: zone,
|
|
71
|
+
reached: queries.funnel_reached(name, since: since, until_time: now),
|
|
72
|
+
stopped: queries.funnel_stopped(name, since: since, until_time: now),
|
|
73
|
+
revisions: queries.by_revision(name, since: since, until_time: now)
|
|
74
|
+
)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# { name => class } of every registered Pgbus::Workflow subclass.
|
|
78
|
+
def registered_workflows = Workflow.registered_classes.to_h { |klass| [klass.name, klass] }
|
|
79
|
+
|
|
80
|
+
# The class a run's workflow_class names, or nil when it does not
|
|
81
|
+
# load or is not a Pgbus::Workflow subclass. Only for names from the
|
|
82
|
+
# registry or the rows.
|
|
83
|
+
def insights_class(name)
|
|
84
|
+
registered_workflows.fetch(name) do
|
|
85
|
+
klass = name.safe_constantize
|
|
86
|
+
klass if klass.is_a?(Class) && klass < Workflow
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# { workflow_class => runs needing a person } (stuck_reason_counts),
|
|
91
|
+
# of every class or of +workflow_class+ only.
|
|
92
|
+
def attention_by_class(workflow_class: nil)
|
|
93
|
+
counts = WorkflowEntry.stuck_reason_counts(**attention_thresholds, workflow_class: workflow_class)
|
|
94
|
+
counts.each_with_object(Hash.new(0)) { |((klass, _), runs), sums| sums[klass] += runs }
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Web
|
|
5
|
+
class DataSource
|
|
6
|
+
# The Workflows pages' reads (issue #542): the list with its filters,
|
|
7
|
+
# one run's detail (attempts, signals, audit trail, what can be done to
|
|
8
|
+
# it), the needs-attention groups and the home-page counts. An install
|
|
9
|
+
# without the workflow tables (pgbus:add_workflows not run) gets empty
|
|
10
|
+
# lists and zero counts, not an error per refresh.
|
|
11
|
+
module WorkflowList
|
|
12
|
+
STATUSES = (%w[ongoing] + Workflow::States::WORKFLOW + %w[all]).freeze
|
|
13
|
+
FILTERS = %i[workflow_class step error_class subject_type subject_id workflow_id created_from created_to
|
|
14
|
+
ended_from ended_to].freeze
|
|
15
|
+
STEP_HISTORY = 100
|
|
16
|
+
EVENT_HISTORY = 200
|
|
17
|
+
# The pause events the timeline pairs, read on their own so a long
|
|
18
|
+
# audit trail never drops a run's earliest pauses from the chart.
|
|
19
|
+
PAUSE_HISTORY = 1_000
|
|
20
|
+
WAIT_TIMEOUT_HISTORY = 200
|
|
21
|
+
SIGNAL_HISTORY = 100
|
|
22
|
+
RERUN_LINKS = 10
|
|
23
|
+
ATTENTION_GROUPS = 200
|
|
24
|
+
|
|
25
|
+
# status – one of STATUSES ("ongoing": ready, performing, waiting or
|
|
26
|
+
# paused). Filters (FILTERS) combine with AND. An unknown status or
|
|
27
|
+
# filter raises ArgumentError (the controller answers 400).
|
|
28
|
+
def workflows(status: "ongoing", page: 1, per_page: 25, **filters)
|
|
29
|
+
check_workflow_query!(status, filters)
|
|
30
|
+
return [] unless Workflow::Sweep.tables_ready?
|
|
31
|
+
|
|
32
|
+
fetch_workflows(status, filters, page, per_page)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def workflows_count(status: "ongoing", **filters)
|
|
36
|
+
check_workflow_query!(status, filters)
|
|
37
|
+
return ListCounts::EMPTY_COUNT unless Workflow::Sweep.tables_ready?
|
|
38
|
+
|
|
39
|
+
count_workflows(status, filters)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def workflow_detail(workflow_id)
|
|
43
|
+
return nil unless Workflow::Sweep.tables_ready?
|
|
44
|
+
|
|
45
|
+
record = WorkflowEntry.find_by(workflow_id: workflow_id)
|
|
46
|
+
return nil unless record
|
|
47
|
+
|
|
48
|
+
steps = WorkflowStep.where(workflow_id: workflow_id).order(id: :desc).limit(STEP_HISTORY).to_a
|
|
49
|
+
progress = batch_progress([step_attribute(steps.first, :batch_id)].compact)
|
|
50
|
+
late = late_since(steps.first&.status == "waiting" ? [steps.first] : [])
|
|
51
|
+
workflow_row(record, steps.first, progress, late).merge(workflow_extras(record, steps))
|
|
52
|
+
rescue StandardError => e
|
|
53
|
+
Pgbus.logger.error { "[Pgbus::Web] Error fetching workflow #{workflow_id}: #{e.class}: #{e.message}" }
|
|
54
|
+
nil
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def ongoing_workflows_count = workflow_status_count(Workflow::States::ONGOING)
|
|
58
|
+
|
|
59
|
+
def paused_workflows_count = workflow_status_count("paused")
|
|
60
|
+
|
|
61
|
+
def workflow_attention_count
|
|
62
|
+
return 0 unless Workflow::Sweep.tables_ready?
|
|
63
|
+
|
|
64
|
+
WorkflowEntry.stuck_reason_counts(**attention_thresholds).values.sum
|
|
65
|
+
rescue StandardError => e
|
|
66
|
+
Pgbus.logger.error { "[Pgbus::Web] Error counting stuck workflows: #{e.class}: #{e.message}" }
|
|
67
|
+
0
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# The stuck runs grouped by (class, step, reason, error class), each
|
|
71
|
+
# with the status its runs are in and the actions that status allows;
|
|
72
|
+
# with +workflow_class+, that class's groups only (issue #549).
|
|
73
|
+
def workflow_attention_groups(workflow_class: nil)
|
|
74
|
+
return [] unless Workflow::Sweep.tables_ready?
|
|
75
|
+
|
|
76
|
+
groups = WorkflowEntry.attention_groups(**attention_thresholds, limit: ATTENTION_GROUPS, workflow_class: workflow_class)
|
|
77
|
+
groups.map do |group|
|
|
78
|
+
status = WorkflowActionPlan.group_status(group[:reason])
|
|
79
|
+
group.merge(status: status, available_actions: Workflow::States.available_actions(status))
|
|
80
|
+
end
|
|
81
|
+
rescue StandardError => e
|
|
82
|
+
Pgbus.logger.error { "[Pgbus::Web] Error fetching stuck workflows: #{e.class}: #{e.message}" }
|
|
83
|
+
[]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
private
|
|
87
|
+
|
|
88
|
+
def check_workflow_query!(status, filters)
|
|
89
|
+
raise ArgumentError, "unknown workflow status: #{status.inspect}" unless STATUSES.include?(status.to_s)
|
|
90
|
+
|
|
91
|
+
unknown = filters.keys - FILTERS
|
|
92
|
+
raise ArgumentError, "unknown workflow filter(s): #{unknown.join(", ")}" if unknown.any?
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def fetch_workflows(status, filters, page, per_page)
|
|
96
|
+
records = workflow_scope(status, filters).order(created_at: :desc, id: :desc)
|
|
97
|
+
.limit(per_page).offset((page - 1) * per_page).to_a
|
|
98
|
+
latest = latest_steps(records.map(&:workflow_id))
|
|
99
|
+
waiting = latest.values.select { |step| step.status == "waiting" }
|
|
100
|
+
progress = batch_progress(waiting.filter_map { |step| step_attribute(step, :batch_id) })
|
|
101
|
+
late = late_since(waiting)
|
|
102
|
+
records.map { |record| workflow_row(record, latest[record.workflow_id], progress, late) }
|
|
103
|
+
rescue StandardError => e
|
|
104
|
+
Pgbus.logger.error { "[Pgbus::Web] Error fetching workflows: #{e.class}: #{e.message}" }
|
|
105
|
+
[]
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def count_workflows(status, filters)
|
|
109
|
+
rows = workflow_scope(status, filters).limit(ListCounts::COUNT_CAP + 1).count
|
|
110
|
+
ListCounts::Count.new(total: [rows, ListCounts::COUNT_CAP].min, capped: rows > ListCounts::COUNT_CAP)
|
|
111
|
+
rescue StandardError => e
|
|
112
|
+
Pgbus.logger.error { "[Pgbus::Web] Error counting workflows: #{e.class}: #{e.message}" }
|
|
113
|
+
ListCounts::EMPTY_COUNT
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def workflow_scope(status, filters)
|
|
117
|
+
scope = WorkflowEntry.all
|
|
118
|
+
status = status.to_s
|
|
119
|
+
scope = scope.where(status: status == "ongoing" ? Workflow::States::ONGOING : status) unless status == "all"
|
|
120
|
+
filters = filters.compact
|
|
121
|
+
{ workflow_class: :workflow_class, step: :current_step, error_class: :error_class, subject_type: :subject_type,
|
|
122
|
+
subject_id: :subject_id, workflow_id: :workflow_id }.each do |filter, column|
|
|
123
|
+
scope = scope.where(column => filters[filter]) if filters.key?(filter)
|
|
124
|
+
end
|
|
125
|
+
scope = scope.where(created_at: filters[:created_from]..) if filters[:created_from]
|
|
126
|
+
scope = scope.where(created_at: ..filters[:created_to]) if filters[:created_to]
|
|
127
|
+
ended_scope(scope, filters)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# When a finished or cancelled run ended (issue #549): what the
|
|
131
|
+
# insights' "finished / cancelled in the window" counts link to. A run
|
|
132
|
+
# ends in the UPDATE that sets its updated_at, so the updated_at bound
|
|
133
|
+
# keeps every match and lets idx_pgbus_workflows_status serve it.
|
|
134
|
+
def ended_scope(scope, filters)
|
|
135
|
+
table = WorkflowEntry.arel_table
|
|
136
|
+
ended_at = Arel::Nodes::NamedFunction.new("COALESCE", [table[:finished_at], table[:cancelled_at]])
|
|
137
|
+
if filters[:ended_from]
|
|
138
|
+
scope = scope.where(table[:updated_at].gteq(filters[:ended_from])).where(ended_at.gteq(filters[:ended_from]))
|
|
139
|
+
end
|
|
140
|
+
scope = scope.where(ended_at.lteq(filters[:ended_to])) if filters[:ended_to]
|
|
141
|
+
scope
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# { workflow_id => its newest step row }, one query for the page.
|
|
145
|
+
def latest_steps(workflow_ids)
|
|
146
|
+
return {} if workflow_ids.empty?
|
|
147
|
+
|
|
148
|
+
WorkflowStep.where(workflow_id: workflow_ids).select("DISTINCT ON (workflow_id) #{WorkflowStep.table_name}.*")
|
|
149
|
+
.order(Arel.sql("workflow_id, id DESC")).index_by(&:workflow_id)
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# { batch_id => { done:, total: } } for waiting fan-out steps, one query.
|
|
153
|
+
def batch_progress(batch_ids)
|
|
154
|
+
return {} if batch_ids.empty?
|
|
155
|
+
|
|
156
|
+
BatchEntry.where(batch_id: batch_ids).to_a.to_h do |batch|
|
|
157
|
+
[batch.batch_id, { done: batch.completed_jobs.to_i + failure_job_count(batch), total: batch.total_jobs.to_i }]
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# { step_id => the first deadline its wait missed } for waiting steps
|
|
162
|
+
# with a deadline, one query: the first wait_timed_out event of each.
|
|
163
|
+
# Only on_timeout: :notify leaves a step waiting after that event
|
|
164
|
+
# (issue #565): every other on_timeout ends the attempt.
|
|
165
|
+
def late_since(waiting)
|
|
166
|
+
steps = waiting.select { |step| step_attribute(step, :timeout_at) }
|
|
167
|
+
return {} if steps.empty?
|
|
168
|
+
|
|
169
|
+
WorkflowEvent.where(workflow_id: steps.map(&:workflow_id), step_id: steps.map(&:id),
|
|
170
|
+
event: WorkflowTimeline::WAIT_TIMED_OUT)
|
|
171
|
+
.select("DISTINCT ON (step_id) step_id, details, created_at").order(Arel.sql("step_id, id"))
|
|
172
|
+
.to_h { |event| [event.step_id, missed_deadline(event)] }
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# The deadline the event reports; when it went out, for an event
|
|
176
|
+
# written without one.
|
|
177
|
+
def missed_deadline(event)
|
|
178
|
+
deadline = event.details.to_h["timeout_at"]
|
|
179
|
+
deadline ? Time.iso8601(deadline) : event.created_at
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# The waiting step's on_timeout as its class declares it: nil without
|
|
183
|
+
# a wait_timeout:, or when the class is not loaded in this process.
|
|
184
|
+
def wait_on_timeout(record, step)
|
|
185
|
+
return nil unless step_attribute(step, :timeout_at)
|
|
186
|
+
|
|
187
|
+
definition = record.workflow_klass&.workflow_steps&.fetch(step.step_name)
|
|
188
|
+
definition.on_timeout.to_s if definition.is_a?(Workflow::StepDefinition)
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def workflow_row(record, step, progress, late = {})
|
|
192
|
+
waiting = step&.status == "waiting"
|
|
193
|
+
{ workflow_id: record.workflow_id, workflow_class: record.workflow_class, subject_type: record.subject_type,
|
|
194
|
+
subject_id: record.subject_id, status: record.status, current_step: record.current_step,
|
|
195
|
+
current_attempt: step&.attempt, step_status: step&.status,
|
|
196
|
+
waiting_for: (step_attribute(step, :wait_for) if waiting),
|
|
197
|
+
waiting_since: (step_attribute(step, :waiting_since) if waiting),
|
|
198
|
+
timeout_at: (step_attribute(step, :timeout_at) if waiting),
|
|
199
|
+
on_timeout: (wait_on_timeout(record, step) if waiting), late_since: (late[step.id] if waiting),
|
|
200
|
+
batch_id: step_attribute(step, :batch_id),
|
|
201
|
+
batch_progress: (progress[step_attribute(step, :batch_id)] if waiting),
|
|
202
|
+
scheduled_at: (step.scheduled_at if step&.status == "scheduled"),
|
|
203
|
+
error_class: record.error_class, error_message: record.error_message,
|
|
204
|
+
paused_at: record.paused_at, finished_at: record.finished_at, cancelled_at: record.cancelled_at,
|
|
205
|
+
rerun_of_workflow_id: record.rerun_of_workflow_id, created_at: record.created_at, updated_at: record.updated_at }
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# The waiting-step columns exist only once pgbus:add_workflow_signals ran.
|
|
209
|
+
def step_attribute(step, name)
|
|
210
|
+
step.public_send(name) if step&.has_attribute?(name)
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def workflow_extras(record, steps)
|
|
214
|
+
step_hashes = steps.map { |step| workflow_step_hash(step) }
|
|
215
|
+
events = WorkflowEvent.where(workflow_id: record.workflow_id).order(id: :desc).limit(EVENT_HISTORY).map do |event|
|
|
216
|
+
{ id: event.id, event: event.event, actor: event.actor, reason: event.reason, step_name: event.step_name,
|
|
217
|
+
attempt: event.attempt, details: event.details, created_at: event.created_at }
|
|
218
|
+
end
|
|
219
|
+
klass = record.workflow_klass
|
|
220
|
+
resume_action, resume_step = WorkflowActionPlan.resume_plan(record, steps.first, klass)
|
|
221
|
+
now = Time.current
|
|
222
|
+
{ arguments: WorkflowPayload.display(record.arguments),
|
|
223
|
+
steps: step_hashes.map { |step| step.merge(durations: WorkflowTimeline.durations(step, now: now)) },
|
|
224
|
+
signals: workflow_signals(record.workflow_id), events: events,
|
|
225
|
+
available_actions: Workflow::States.available_actions(record.status),
|
|
226
|
+
definition_outdated: record.definition_outdated?, class_loaded: !klass.nil?,
|
|
227
|
+
step_names: klass ? klass.workflow_steps.names : [], resume_action: resume_action, resume_step: resume_step,
|
|
228
|
+
signal_names: WorkflowActionPlan.signal_names(steps.first, klass), reruns: workflow_reruns(record.workflow_id),
|
|
229
|
+
timeline: WorkflowTimeline.chart(step_hashes, timeline_events(record.workflow_id), status: record.status, now: now) }
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
def workflow_step_hash(step)
|
|
233
|
+
{ id: step.id, step_name: step.step_name, attempt: step.attempt, status: step.status, outcome: step.outcome,
|
|
234
|
+
created_at: step.created_at, scheduled_at: step.scheduled_at, enqueued_at: step.enqueued_at,
|
|
235
|
+
started_at: step.started_at, finished_at: step.finished_at, job_id: step.job_id, msg_id: step.msg_id,
|
|
236
|
+
worker: step.worker, app_revision: step.app_revision, error_class: step.error_class,
|
|
237
|
+
error_message: step.error_message, error_backtrace: step.error_backtrace, error_causes: step.error_causes || [],
|
|
238
|
+
output: step.output && WorkflowPayload.display(step.output),
|
|
239
|
+
waiting_since: step_attribute(step, :waiting_since), wait_for: step_attribute(step, :wait_for),
|
|
240
|
+
timeout_at: step_attribute(step, :timeout_at), batch_id: step_attribute(step, :batch_id),
|
|
241
|
+
signal_id: step_attribute(step, :signal_id) }
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# The pauses (oldest first) and the newest wait_timed_out events the
|
|
245
|
+
# timeline marks, read apart so a wait late for weeks cannot crowd out
|
|
246
|
+
# the pauses.
|
|
247
|
+
def timeline_events(workflow_id)
|
|
248
|
+
pauses = WorkflowEvent.where(workflow_id: workflow_id, event: WorkflowTimeline::PAUSE_EVENTS).order(:id)
|
|
249
|
+
.limit(PAUSE_HISTORY).map { |event| { event: event.event, created_at: event.created_at } }
|
|
250
|
+
timeouts = WorkflowEvent.where(workflow_id: workflow_id, event: WorkflowTimeline::WAIT_TIMED_OUT)
|
|
251
|
+
.order(id: :desc).limit(WAIT_TIMEOUT_HISTORY).map do |event|
|
|
252
|
+
{ event: event.event, created_at: event.created_at, step_name: event.step_name, attempt: event.attempt,
|
|
253
|
+
details: event.details }
|
|
254
|
+
end
|
|
255
|
+
pauses + timeouts
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def workflow_signals(workflow_id)
|
|
259
|
+
return [] unless Workflow::Sweep.waiting_ready?
|
|
260
|
+
|
|
261
|
+
WorkflowSignal.where(workflow_id: workflow_id).order(id: :desc).limit(SIGNAL_HISTORY).map do |signal|
|
|
262
|
+
{ id: signal.id, name: signal.name, status: signal.status, idempotency_key: signal.idempotency_key,
|
|
263
|
+
payload: WorkflowPayload.display(signal.read_attribute(:payload)), created_at: signal.created_at,
|
|
264
|
+
claimed_at: signal.claimed_at, consumed_at: signal.consumed_at }
|
|
265
|
+
end
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
def workflow_reruns(workflow_id)
|
|
269
|
+
WorkflowEntry.where(rerun_of_workflow_id: workflow_id).order(created_at: :desc).limit(RERUN_LINKS)
|
|
270
|
+
.pluck(:workflow_id, :status, :created_at).map do |id, status, created_at|
|
|
271
|
+
{ workflow_id: id, status: status, created_at: created_at }
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
def workflow_status_count(status)
|
|
276
|
+
return 0 unless Workflow::Sweep.tables_ready?
|
|
277
|
+
|
|
278
|
+
WorkflowEntry.where(status: status).count
|
|
279
|
+
rescue StandardError => e
|
|
280
|
+
Pgbus.logger.error { "[Pgbus::Web] Error counting workflows: #{e.class}: #{e.message}" }
|
|
281
|
+
0
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
def attention_thresholds(config: Pgbus.configuration)
|
|
285
|
+
waiting = Workflow::Sweep.waiting_ready? ? config.workflow_waiting_visibility_threshold : nil
|
|
286
|
+
{ threshold: config.workflow_stall_threshold, waiting_threshold: waiting }
|
|
287
|
+
end
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
end
|
|
291
|
+
end
|