pgbus 0.19.3 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/README.md +104 -9
- data/Rakefile +6 -1
- data/app/controllers/pgbus/dashboard_controller.rb +2 -0
- data/app/controllers/pgbus/workflow_definitions_controller.rb +28 -0
- data/app/controllers/pgbus/workflows_controller.rb +197 -0
- data/app/frontend/pgbus/modules/charts.js +130 -4
- data/app/frontend/pgbus/modules/charts.min.js +1 -1
- data/app/frontend/pgbus/modules/charts.min.js.map +1 -1
- data/app/frontend/pgbus/style.css +1 -1
- data/app/helpers/pgbus/button_helper.rb +4 -2
- data/app/helpers/pgbus/workflow_insights_helper.rb +129 -0
- data/app/helpers/pgbus/workflows_helper.rb +165 -0
- data/app/models/pgbus/workflow_entry.rb +172 -0
- data/app/models/pgbus/workflow_event.rb +43 -0
- data/app/models/pgbus/workflow_signal.rb +77 -0
- data/app/models/pgbus/workflow_step.rb +142 -0
- data/app/views/layouts/pgbus/application.html.erb +2 -0
- data/app/views/pgbus/dashboard/_stats_cards.html.erb +17 -0
- data/app/views/pgbus/workflows/_actions.html.erb +59 -0
- data/app/views/pgbus/workflows/_events_table.html.erb +24 -0
- data/app/views/pgbus/workflows/_filters.html.erb +29 -0
- data/app/views/pgbus/workflows/_signal_form.html.erb +31 -0
- data/app/views/pgbus/workflows/_signals_table.html.erb +40 -0
- data/app/views/pgbus/workflows/_status_filter.html.erb +12 -0
- data/app/views/pgbus/workflows/_steps_table.html.erb +79 -0
- data/app/views/pgbus/workflows/_timeline.html.erb +39 -0
- data/app/views/pgbus/workflows/_workflows_table.html.erb +59 -0
- data/app/views/pgbus/workflows/attention.html.erb +88 -0
- data/app/views/pgbus/workflows/definitions/_by_revision.html.erb +52 -0
- data/app/views/pgbus/workflows/definitions/_duration_table.html.erb +18 -0
- data/app/views/pgbus/workflows/definitions/_funnel.html.erb +49 -0
- data/app/views/pgbus/workflows/definitions/_pipeline.html.erb +16 -0
- data/app/views/pgbus/workflows/definitions/_step_card.html.erb +31 -0
- data/app/views/pgbus/workflows/definitions/_throughput_table.html.erb +32 -0
- data/app/views/pgbus/workflows/definitions/_window.html.erb +17 -0
- data/app/views/pgbus/workflows/definitions/index.html.erb +71 -0
- data/app/views/pgbus/workflows/definitions/not_found.html.erb +5 -0
- data/app/views/pgbus/workflows/definitions/show.html.erb +80 -0
- data/app/views/pgbus/workflows/index.html.erb +15 -0
- data/app/views/pgbus/workflows/not_found.html.erb +5 -0
- data/app/views/pgbus/workflows/show.html.erb +93 -0
- data/config/i18n-tasks.yml +6 -0
- data/config/locales/da.yml +442 -0
- data/config/locales/de.yml +442 -0
- data/config/locales/en.yml +442 -0
- data/config/locales/es.yml +442 -0
- data/config/locales/fi.yml +442 -0
- data/config/locales/fr.yml +442 -0
- data/config/locales/it.yml +442 -0
- data/config/locales/ja.yml +442 -0
- data/config/locales/nb.yml +442 -0
- data/config/locales/nl.yml +442 -0
- data/config/locales/pt.yml +442 -0
- data/config/locales/sv.yml +442 -0
- data/config/routes.rb +26 -0
- data/lib/generators/pgbus/add_workflow_signals_generator.rb +56 -0
- data/lib/generators/pgbus/add_workflows_generator.rb +54 -0
- data/lib/generators/pgbus/templates/add_workflow_signals.rb.erb +51 -0
- data/lib/generators/pgbus/templates/add_workflows.rb.erb +150 -0
- data/lib/generators/pgbus/templates/migration.rb.erb +127 -0
- data/lib/pgbus/active_job/adapter.rb +67 -26
- data/lib/pgbus/active_job/batch_id.rb +23 -1
- data/lib/pgbus/active_job/delivery.rb +49 -0
- data/lib/pgbus/active_job/executor.rb +3 -1
- data/lib/pgbus/batch/builder.rb +1 -0
- data/lib/pgbus/batch.rb +35 -3
- data/lib/pgbus/configuration/workflow_settings.rb +99 -0
- data/lib/pgbus/configuration.rb +21 -3
- data/lib/pgbus/generators/migration_detector.rb +40 -0
- data/lib/pgbus/instrumentation.rb +32 -0
- data/lib/pgbus/integrations/appsignal/dashboard.json +128 -1
- data/lib/pgbus/integrations/appsignal/event_span.rb +43 -0
- data/lib/pgbus/integrations/appsignal/job_span.rb +47 -0
- data/lib/pgbus/integrations/appsignal/span_subscriber.rb +128 -0
- data/lib/pgbus/integrations/appsignal/subscriber.rb +50 -100
- data/lib/pgbus/mcp/rack_app.rb +7 -2
- data/lib/pgbus/mcp/runner.rb +2 -1
- data/lib/pgbus/mcp/serialization.rb +25 -0
- data/lib/pgbus/mcp/server.rb +53 -10
- data/lib/pgbus/mcp/tools/deploy_tasks_tool.rb +1 -10
- data/lib/pgbus/mcp/tools/workflow_cancel_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_definitions_tool.rb +36 -0
- data/lib/pgbus/mcp/tools/workflow_detail_tool.rb +38 -0
- data/lib/pgbus/mcp/tools/workflow_resume_at_tool.rb +32 -0
- data/lib/pgbus/mcp/tools/workflow_resume_tool.rb +24 -0
- data/lib/pgbus/mcp/tools/workflow_run_now_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_signal_tool.rb +77 -0
- data/lib/pgbus/mcp/tools/workflow_skip_step_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_stats_tool.rb +42 -0
- data/lib/pgbus/mcp/tools/workflows_attention_tool.rb +42 -0
- data/lib/pgbus/mcp/tools/workflows_tool.rb +60 -0
- data/lib/pgbus/mcp/workflow_insights_payload.rb +15 -0
- data/lib/pgbus/mcp/workflow_payloads.rb +53 -0
- data/lib/pgbus/mcp/write_tool.rb +134 -0
- data/lib/pgbus/mcp.rb +15 -0
- data/lib/pgbus/metrics/backends/statsd.rb +6 -0
- data/lib/pgbus/metrics/subscriber.rb +14 -1
- data/lib/pgbus/metrics/workflow_metrics.rb +65 -0
- data/lib/pgbus/process/dispatcher.rb +26 -0
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source/workflow_actions.rb +109 -0
- data/lib/pgbus/web/data_source/workflow_insights.rb +99 -0
- data/lib/pgbus/web/data_source/workflow_list.rb +248 -0
- data/lib/pgbus/web/data_source.rb +3 -0
- data/lib/pgbus/web/workflow_action_plan.rb +62 -0
- data/lib/pgbus/web/workflow_insights_queries.rb +250 -0
- data/lib/pgbus/web/workflow_insights_report.rb +171 -0
- data/lib/pgbus/web/workflow_payload.rb +36 -0
- data/lib/pgbus/web/workflow_query.rb +82 -0
- data/lib/pgbus/web/workflow_state.rb +76 -0
- data/lib/pgbus/web/workflow_timeline.rb +113 -0
- data/lib/pgbus/web/workflow_window.rb +51 -0
- data/lib/pgbus/workflow/batch_finished_job.rb +21 -0
- data/lib/pgbus/workflow/dsl.rb +105 -0
- data/lib/pgbus/workflow/error_capture.rb +63 -0
- data/lib/pgbus/workflow/exception_policy.rb +170 -0
- data/lib/pgbus/workflow/fan_out.rb +157 -0
- data/lib/pgbus/workflow/fan_out_definition.rb +41 -0
- data/lib/pgbus/workflow/flow_control.rb +94 -0
- data/lib/pgbus/workflow/instance.rb +183 -0
- data/lib/pgbus/workflow/outside_verbs.rb +138 -0
- data/lib/pgbus/workflow/payload.rb +53 -0
- data/lib/pgbus/workflow/registry.rb +53 -0
- data/lib/pgbus/workflow/removed_step.rb +14 -0
- data/lib/pgbus/workflow/runner.rb +218 -0
- data/lib/pgbus/workflow/signal_matcher.rb +53 -0
- data/lib/pgbus/workflow/signals.rb +132 -0
- data/lib/pgbus/workflow/states.rb +119 -0
- data/lib/pgbus/workflow/step_context.rb +58 -0
- data/lib/pgbus/workflow/step_definition.rb +131 -0
- data/lib/pgbus/workflow/step_execution.rb +242 -0
- data/lib/pgbus/workflow/step_job.rb +28 -0
- data/lib/pgbus/workflow/step_outcome.rb +151 -0
- data/lib/pgbus/workflow/steps.rb +103 -0
- data/lib/pgbus/workflow/sweep.rb +156 -0
- data/lib/pgbus/workflow/test_helpers.rb +152 -0
- data/lib/pgbus/workflow/wait_timeout.rb +73 -0
- data/lib/pgbus/workflow/waiting.rb +82 -0
- data/lib/pgbus/workflow.rb +183 -0
- data/sig/pgbus/configuration.rbs +22 -0
- metadata +95 -1
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Web
|
|
5
|
+
class DataSource
|
|
6
|
+
# The Workflows dashboard's actions (issue #542). Each goes through the
|
|
7
|
+
# run's own outside verb (Pgbus::Workflow#resume! and friends), so the
|
|
8
|
+
# States table and the row lock decide, and returns the real outcome:
|
|
9
|
+
#
|
|
10
|
+
# { ok: true } it happened
|
|
11
|
+
# { ok: false, code:, reason:, status: } it did not, and why
|
|
12
|
+
#
|
|
13
|
+
# code is :not_allowed (the run is no longer in a state that allows it,
|
|
14
|
+
# usually a lost race: status says where it is now), :not_found,
|
|
15
|
+
# :invalid (an unknown step, a payload over the size cap) or :failed
|
|
16
|
+
# (anything else: logged). actor is who it is recorded as on the run's
|
|
17
|
+
# audit trail; reason is the operator's note.
|
|
18
|
+
module WorkflowActions
|
|
19
|
+
# At most this many runs per bulk call; a bigger group takes another press.
|
|
20
|
+
BULK_CAP = 500
|
|
21
|
+
# The outcomes a bulk call counts as "had already moved on".
|
|
22
|
+
MOVED_ON = %i[not_allowed not_found].freeze
|
|
23
|
+
|
|
24
|
+
def resume_workflow(workflow_id, actor:, reason: nil)
|
|
25
|
+
workflow_action(workflow_id) { |wf| wf.resume!(actor: actor, reason: reason) }
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def cancel_workflow(workflow_id, actor:, reason: nil)
|
|
29
|
+
workflow_action(workflow_id) { |wf| wf.cancel!(actor: actor, reason: reason) }
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def skip_workflow_step(workflow_id, actor:, reason: nil)
|
|
33
|
+
workflow_action(workflow_id) { |wf| wf.skip!(actor: actor, reason: reason) }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def resume_workflow_at(workflow_id, step:, actor:, reason: nil)
|
|
37
|
+
workflow_action(workflow_id) { |wf| wf.resume_at!(step, actor: actor, reason: reason) }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def run_workflow_step_now(workflow_id, actor:, reason: nil)
|
|
41
|
+
workflow_action(workflow_id) { |wf| wf.run_now!(actor: actor, reason: reason) }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# A redelivery with the same idempotency_key stores nothing: ok, with
|
|
45
|
+
# duplicate: true. The actor and reason are on the signal_received event.
|
|
46
|
+
def signal_workflow(workflow_id, name:, payload:, idempotency_key:, actor:, reason: nil)
|
|
47
|
+
duplicate = false
|
|
48
|
+
result = workflow_action(workflow_id) do |wf|
|
|
49
|
+
signal = wf.signal!(name, payload: payload, idempotency_key: idempotency_key, actor: actor, reason: reason)
|
|
50
|
+
duplicate = signal.duplicate_delivery?
|
|
51
|
+
end
|
|
52
|
+
result[:ok] ? result.merge(duplicate: duplicate) : result
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# The new run's workflow_id on success.
|
|
56
|
+
def rerun_workflow(workflow_id, actor:, reason: nil)
|
|
57
|
+
rerun = nil
|
|
58
|
+
result = workflow_action(workflow_id) { |wf| rerun = wf.rerun!(actor: actor, reason: reason) }
|
|
59
|
+
result[:ok] ? result.merge(workflow_id: rerun.workflow_id) : result
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# +group+ is a needs-attention group (workflow_class, step_name,
|
|
63
|
+
# reason, error_class). Each of its runs, at most BULK_CAP, through its
|
|
64
|
+
# own guarded transition: one that moved on in the meantime is
|
|
65
|
+
# skipped. { ok:, done:, skipped:, failed:, failure: }, or ok: false
|
|
66
|
+
# with a reason when the group could not be read.
|
|
67
|
+
def bulk_resume_workflows(group:, actor:, reason: nil)
|
|
68
|
+
bulk_workflow_action(group) { |id| resume_workflow(id, actor: actor, reason: reason) }
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def bulk_cancel_workflows(group:, actor:, reason: nil)
|
|
72
|
+
bulk_workflow_action(group) { |id| cancel_workflow(id, actor: actor, reason: reason) }
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
private
|
|
76
|
+
|
|
77
|
+
def workflow_action(workflow_id)
|
|
78
|
+
workflow = Workflow.find(workflow_id)
|
|
79
|
+
return { ok: false, code: :not_found, reason: "workflow #{workflow_id} not found" } unless workflow
|
|
80
|
+
|
|
81
|
+
yield workflow
|
|
82
|
+
{ ok: true }
|
|
83
|
+
rescue Workflow::InvalidTransition => e
|
|
84
|
+
{ ok: false, code: :not_allowed, reason: e.message, status: WorkflowEntry.where(workflow_id: workflow_id).pick(:status) }
|
|
85
|
+
rescue ActiveRecord::RecordNotFound => e
|
|
86
|
+
{ ok: false, code: :not_found, reason: e.message }
|
|
87
|
+
rescue Workflow::StepNotDefined, Workflow::PayloadTooLarge, ArgumentError => e
|
|
88
|
+
{ ok: false, code: :invalid, reason: e.message }
|
|
89
|
+
rescue StandardError => e
|
|
90
|
+
Pgbus.logger.error { "[Pgbus::Web] Workflow action on #{workflow_id} failed: #{e.class}: #{e.message}" }
|
|
91
|
+
{ ok: false, code: :failed, reason: "#{e.class}: #{e.message}" }
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# done: changed. skipped: had moved on (no longer in a state that
|
|
95
|
+
# allows it, or gone). failed: anything else, with the first reason.
|
|
96
|
+
def bulk_workflow_action(group, &)
|
|
97
|
+
ids = WorkflowEntry.attention_ids(group, **attention_thresholds, limit: BULK_CAP)
|
|
98
|
+
outcomes = ids.map(&)
|
|
99
|
+
moved_on, failed = outcomes.reject { |outcome| outcome[:ok] }.partition { |outcome| MOVED_ON.include?(outcome[:code]) }
|
|
100
|
+
result = { ok: true, done: outcomes.size - moved_on.size - failed.size, skipped: moved_on.size, failed: failed.size }
|
|
101
|
+
failed.empty? ? result : result.merge(failure: failed.first[:reason])
|
|
102
|
+
rescue StandardError => e
|
|
103
|
+
Pgbus.logger.error { "[Pgbus::Web] Bulk workflow action on #{group.inspect} failed: #{e.class}: #{e.message}" }
|
|
104
|
+
{ ok: false, code: :failed, reason: "#{e.class}: #{e.message}", done: 0, skipped: 0, failed: 0 }
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Web
|
|
5
|
+
class DataSource
|
|
6
|
+
# The workflow insights' reads (issue #549): one row per workflow class
|
|
7
|
+
# for /pgbus/workflows/definitions, and one class's pipeline, step
|
|
8
|
+
# performance, throughput, durations, funnel and per-revision numbers
|
|
9
|
+
# for /pgbus/workflows/definitions/:workflow_class and the MCP tools.
|
|
10
|
+
# Read from the step and run rows inside a WorkflowWindow
|
|
11
|
+
# (WorkflowInsightsQueries); WorkflowInsightsReport shapes the answer.
|
|
12
|
+
#
|
|
13
|
+
# A class is the registered Pgbus::Workflow subclass of that name, or,
|
|
14
|
+
# for a name present in pgbus_workflows, its safe_constantize when that
|
|
15
|
+
# is a Pgbus::Workflow subclass. A name from a URL that is neither is
|
|
16
|
+
# never constantized: the class is unknown.
|
|
17
|
+
module WorkflowInsights
|
|
18
|
+
# A workflow class name: a constant path (OrderWorkflow,
|
|
19
|
+
# Billing::OrderWorkflow). The route constrains to the same shape.
|
|
20
|
+
CLASS_NAME = /\A[A-Z][A-Za-z0-9_]*(?:::[A-Z][A-Za-z0-9_]*)*\z/
|
|
21
|
+
|
|
22
|
+
def workflow_definitions(window:)
|
|
23
|
+
return [] unless Workflow::Sweep.tables_ready?
|
|
24
|
+
|
|
25
|
+
ongoing = WorkflowInsightsQueries.ongoing.group_by { |row| row[:workflow_class] }
|
|
26
|
+
ended = WorkflowInsightsQueries.ended(since: window.since, until_time: window.now).index_by { |row| row[:workflow_class] }
|
|
27
|
+
stuck = attention_by_class
|
|
28
|
+
names = (registered_workflows.keys + WorkflowInsightsQueries.class_names + ongoing.keys + ended.keys + stuck.keys)
|
|
29
|
+
names.uniq.sort.map do |name|
|
|
30
|
+
WorkflowInsightsReport.summary(name, insights_class(name), ongoing.fetch(name, []), ended[name], stuck.fetch(name, 0))
|
|
31
|
+
end
|
|
32
|
+
rescue StandardError => e
|
|
33
|
+
Pgbus.logger.error { "[Pgbus::Web] Error reading workflow definitions: #{e.class}: #{e.message}" }
|
|
34
|
+
[]
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# nil for a name that is not a constant path, that no class or run
|
|
38
|
+
# has, or when the read failed.
|
|
39
|
+
def workflow_definition(workflow_class, window:)
|
|
40
|
+
return nil unless Workflow::Sweep.tables_ready?
|
|
41
|
+
|
|
42
|
+
name = workflow_class.to_s
|
|
43
|
+
return nil unless CLASS_NAME.match?(name)
|
|
44
|
+
|
|
45
|
+
registered = registered_workflows[name]
|
|
46
|
+
return nil unless registered || WorkflowInsightsQueries.class_known?(name)
|
|
47
|
+
|
|
48
|
+
read_definition(name, registered || insights_class(name), window)
|
|
49
|
+
rescue StandardError => e
|
|
50
|
+
Pgbus.logger.error { "[Pgbus::Web] Error reading workflow definition #{workflow_class}: #{e.class}: #{e.message}" }
|
|
51
|
+
nil
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
def read_definition(name, klass, window)
|
|
57
|
+
now = window.now
|
|
58
|
+
zone = Time.zone || ActiveSupport::TimeZone["UTC"]
|
|
59
|
+
since = window.since
|
|
60
|
+
queries = WorkflowInsightsQueries
|
|
61
|
+
WorkflowInsightsReport.detail(
|
|
62
|
+
name, klass,
|
|
63
|
+
window: window, since: since, now: now, ongoing: queries.ongoing(workflow_class: name),
|
|
64
|
+
ended: queries.ended(since: since, until_time: now, workflow_class: name).first,
|
|
65
|
+
attention: attention_by_class(workflow_class: name).fetch(name, 0),
|
|
66
|
+
stats: queries.step_stats(name, since: since, until_time: now),
|
|
67
|
+
histogram: queries.duration_histogram(name, since: since, until_time: now),
|
|
68
|
+
throughput: queries.throughput(name, since: since, until_time: now, bucket: window.bucket_seconds,
|
|
69
|
+
zone: zone.tzinfo.identifier),
|
|
70
|
+
zone: zone,
|
|
71
|
+
reached: queries.funnel_reached(name, since: since, until_time: now),
|
|
72
|
+
stopped: queries.funnel_stopped(name, since: since, until_time: now),
|
|
73
|
+
revisions: queries.by_revision(name, since: since, until_time: now)
|
|
74
|
+
)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# { name => class } of every registered Pgbus::Workflow subclass.
|
|
78
|
+
def registered_workflows = Workflow.registered_classes.to_h { |klass| [klass.name, klass] }
|
|
79
|
+
|
|
80
|
+
# The class a run's workflow_class names, or nil when it does not
|
|
81
|
+
# load or is not a Pgbus::Workflow subclass. Only for names from the
|
|
82
|
+
# registry or the rows.
|
|
83
|
+
def insights_class(name)
|
|
84
|
+
registered_workflows.fetch(name) do
|
|
85
|
+
klass = name.safe_constantize
|
|
86
|
+
klass if klass.is_a?(Class) && klass < Workflow
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# { workflow_class => runs needing a person } (stuck_reason_counts),
|
|
91
|
+
# of every class or of +workflow_class+ only.
|
|
92
|
+
def attention_by_class(workflow_class: nil)
|
|
93
|
+
counts = WorkflowEntry.stuck_reason_counts(**attention_thresholds, workflow_class: workflow_class)
|
|
94
|
+
counts.each_with_object(Hash.new(0)) { |((klass, _), runs), sums| sums[klass] += runs }
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Web
|
|
5
|
+
class DataSource
|
|
6
|
+
# The Workflows pages' reads (issue #542): the list with its filters,
|
|
7
|
+
# one run's detail (attempts, signals, audit trail, what can be done to
|
|
8
|
+
# it), the needs-attention groups and the home-page counts. An install
|
|
9
|
+
# without the workflow tables (pgbus:add_workflows not run) gets empty
|
|
10
|
+
# lists and zero counts, not an error per refresh.
|
|
11
|
+
module WorkflowList
|
|
12
|
+
STATUSES = (%w[ongoing] + Workflow::States::WORKFLOW + %w[all]).freeze
|
|
13
|
+
FILTERS = %i[workflow_class step error_class subject_type subject_id workflow_id created_from created_to
|
|
14
|
+
ended_from ended_to].freeze
|
|
15
|
+
STEP_HISTORY = 100
|
|
16
|
+
EVENT_HISTORY = 200
|
|
17
|
+
# The pause events the timeline pairs, read on their own so a long
|
|
18
|
+
# audit trail never drops a run's earliest pauses from the chart.
|
|
19
|
+
PAUSE_HISTORY = 1_000
|
|
20
|
+
SIGNAL_HISTORY = 100
|
|
21
|
+
RERUN_LINKS = 10
|
|
22
|
+
ATTENTION_GROUPS = 200
|
|
23
|
+
|
|
24
|
+
# status – one of STATUSES ("ongoing": ready, performing, waiting or
|
|
25
|
+
# paused). Filters (FILTERS) combine with AND. An unknown status or
|
|
26
|
+
# filter raises ArgumentError (the controller answers 400).
|
|
27
|
+
def workflows(status: "ongoing", page: 1, per_page: 25, **filters)
|
|
28
|
+
check_workflow_query!(status, filters)
|
|
29
|
+
return [] unless Workflow::Sweep.tables_ready?
|
|
30
|
+
|
|
31
|
+
fetch_workflows(status, filters, page, per_page)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def workflows_count(status: "ongoing", **filters)
|
|
35
|
+
check_workflow_query!(status, filters)
|
|
36
|
+
return ListCounts::EMPTY_COUNT unless Workflow::Sweep.tables_ready?
|
|
37
|
+
|
|
38
|
+
count_workflows(status, filters)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def workflow_detail(workflow_id)
|
|
42
|
+
return nil unless Workflow::Sweep.tables_ready?
|
|
43
|
+
|
|
44
|
+
record = WorkflowEntry.find_by(workflow_id: workflow_id)
|
|
45
|
+
return nil unless record
|
|
46
|
+
|
|
47
|
+
steps = WorkflowStep.where(workflow_id: workflow_id).order(id: :desc).limit(STEP_HISTORY).to_a
|
|
48
|
+
progress = batch_progress([step_attribute(steps.first, :batch_id)].compact)
|
|
49
|
+
workflow_row(record, steps.first, progress).merge(workflow_extras(record, steps))
|
|
50
|
+
rescue StandardError => e
|
|
51
|
+
Pgbus.logger.error { "[Pgbus::Web] Error fetching workflow #{workflow_id}: #{e.class}: #{e.message}" }
|
|
52
|
+
nil
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def ongoing_workflows_count = workflow_status_count(Workflow::States::ONGOING)
|
|
56
|
+
|
|
57
|
+
def paused_workflows_count = workflow_status_count("paused")
|
|
58
|
+
|
|
59
|
+
def workflow_attention_count
|
|
60
|
+
return 0 unless Workflow::Sweep.tables_ready?
|
|
61
|
+
|
|
62
|
+
WorkflowEntry.stuck_reason_counts(**attention_thresholds).values.sum
|
|
63
|
+
rescue StandardError => e
|
|
64
|
+
Pgbus.logger.error { "[Pgbus::Web] Error counting stuck workflows: #{e.class}: #{e.message}" }
|
|
65
|
+
0
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# The stuck runs grouped by (class, step, reason, error class), each
|
|
69
|
+
# with the status its runs are in and the actions that status allows;
|
|
70
|
+
# with +workflow_class+, that class's groups only (issue #549).
|
|
71
|
+
def workflow_attention_groups(workflow_class: nil)
|
|
72
|
+
return [] unless Workflow::Sweep.tables_ready?
|
|
73
|
+
|
|
74
|
+
groups = WorkflowEntry.attention_groups(**attention_thresholds, limit: ATTENTION_GROUPS, workflow_class: workflow_class)
|
|
75
|
+
groups.map do |group|
|
|
76
|
+
status = WorkflowActionPlan.group_status(group[:reason])
|
|
77
|
+
group.merge(status: status, available_actions: Workflow::States.available_actions(status))
|
|
78
|
+
end
|
|
79
|
+
rescue StandardError => e
|
|
80
|
+
Pgbus.logger.error { "[Pgbus::Web] Error fetching stuck workflows: #{e.class}: #{e.message}" }
|
|
81
|
+
[]
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
private
|
|
85
|
+
|
|
86
|
+
def check_workflow_query!(status, filters)
|
|
87
|
+
raise ArgumentError, "unknown workflow status: #{status.inspect}" unless STATUSES.include?(status.to_s)
|
|
88
|
+
|
|
89
|
+
unknown = filters.keys - FILTERS
|
|
90
|
+
raise ArgumentError, "unknown workflow filter(s): #{unknown.join(", ")}" if unknown.any?
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def fetch_workflows(status, filters, page, per_page)
|
|
94
|
+
records = workflow_scope(status, filters).order(created_at: :desc, id: :desc)
|
|
95
|
+
.limit(per_page).offset((page - 1) * per_page).to_a
|
|
96
|
+
latest = latest_steps(records.map(&:workflow_id))
|
|
97
|
+
waiting = latest.values.select { |step| step.status == "waiting" }
|
|
98
|
+
progress = batch_progress(waiting.filter_map { |step| step_attribute(step, :batch_id) })
|
|
99
|
+
records.map { |record| workflow_row(record, latest[record.workflow_id], progress) }
|
|
100
|
+
rescue StandardError => e
|
|
101
|
+
Pgbus.logger.error { "[Pgbus::Web] Error fetching workflows: #{e.class}: #{e.message}" }
|
|
102
|
+
[]
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def count_workflows(status, filters)
|
|
106
|
+
rows = workflow_scope(status, filters).limit(ListCounts::COUNT_CAP + 1).count
|
|
107
|
+
ListCounts::Count.new(total: [rows, ListCounts::COUNT_CAP].min, capped: rows > ListCounts::COUNT_CAP)
|
|
108
|
+
rescue StandardError => e
|
|
109
|
+
Pgbus.logger.error { "[Pgbus::Web] Error counting workflows: #{e.class}: #{e.message}" }
|
|
110
|
+
ListCounts::EMPTY_COUNT
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def workflow_scope(status, filters)
|
|
114
|
+
scope = WorkflowEntry.all
|
|
115
|
+
status = status.to_s
|
|
116
|
+
scope = scope.where(status: status == "ongoing" ? Workflow::States::ONGOING : status) unless status == "all"
|
|
117
|
+
filters = filters.compact
|
|
118
|
+
{ workflow_class: :workflow_class, step: :current_step, error_class: :error_class, subject_type: :subject_type,
|
|
119
|
+
subject_id: :subject_id, workflow_id: :workflow_id }.each do |filter, column|
|
|
120
|
+
scope = scope.where(column => filters[filter]) if filters.key?(filter)
|
|
121
|
+
end
|
|
122
|
+
scope = scope.where(created_at: filters[:created_from]..) if filters[:created_from]
|
|
123
|
+
scope = scope.where(created_at: ..filters[:created_to]) if filters[:created_to]
|
|
124
|
+
ended_scope(scope, filters)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# When a finished or cancelled run ended (issue #549): what the
|
|
128
|
+
# insights' "finished / cancelled in the window" counts link to. A run
|
|
129
|
+
# ends in the UPDATE that sets its updated_at, so the updated_at bound
|
|
130
|
+
# keeps every match and lets idx_pgbus_workflows_status serve it.
|
|
131
|
+
def ended_scope(scope, filters)
|
|
132
|
+
table = WorkflowEntry.arel_table
|
|
133
|
+
ended_at = Arel::Nodes::NamedFunction.new("COALESCE", [table[:finished_at], table[:cancelled_at]])
|
|
134
|
+
if filters[:ended_from]
|
|
135
|
+
scope = scope.where(table[:updated_at].gteq(filters[:ended_from])).where(ended_at.gteq(filters[:ended_from]))
|
|
136
|
+
end
|
|
137
|
+
scope = scope.where(ended_at.lteq(filters[:ended_to])) if filters[:ended_to]
|
|
138
|
+
scope
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# { workflow_id => its newest step row }, one query for the page.
|
|
142
|
+
def latest_steps(workflow_ids)
|
|
143
|
+
return {} if workflow_ids.empty?
|
|
144
|
+
|
|
145
|
+
WorkflowStep.where(workflow_id: workflow_ids).select("DISTINCT ON (workflow_id) #{WorkflowStep.table_name}.*")
|
|
146
|
+
.order(Arel.sql("workflow_id, id DESC")).index_by(&:workflow_id)
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# { batch_id => { done:, total: } } for waiting fan-out steps, one query.
|
|
150
|
+
def batch_progress(batch_ids)
|
|
151
|
+
return {} if batch_ids.empty?
|
|
152
|
+
|
|
153
|
+
BatchEntry.where(batch_id: batch_ids).to_a.to_h do |batch|
|
|
154
|
+
[batch.batch_id, { done: batch.completed_jobs.to_i + failure_job_count(batch), total: batch.total_jobs.to_i }]
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def workflow_row(record, step, progress)
|
|
159
|
+
waiting = step&.status == "waiting"
|
|
160
|
+
{ workflow_id: record.workflow_id, workflow_class: record.workflow_class, subject_type: record.subject_type,
|
|
161
|
+
subject_id: record.subject_id, status: record.status, current_step: record.current_step,
|
|
162
|
+
current_attempt: step&.attempt, step_status: step&.status,
|
|
163
|
+
waiting_for: (step_attribute(step, :wait_for) if waiting),
|
|
164
|
+
waiting_since: (step_attribute(step, :waiting_since) if waiting),
|
|
165
|
+
timeout_at: (step_attribute(step, :timeout_at) if waiting),
|
|
166
|
+
batch_id: step_attribute(step, :batch_id),
|
|
167
|
+
batch_progress: (progress[step_attribute(step, :batch_id)] if waiting),
|
|
168
|
+
scheduled_at: (step.scheduled_at if step&.status == "scheduled"),
|
|
169
|
+
error_class: record.error_class, error_message: record.error_message,
|
|
170
|
+
paused_at: record.paused_at, finished_at: record.finished_at, cancelled_at: record.cancelled_at,
|
|
171
|
+
rerun_of_workflow_id: record.rerun_of_workflow_id, created_at: record.created_at, updated_at: record.updated_at }
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# The waiting-step columns exist only once pgbus:add_workflow_signals ran.
|
|
175
|
+
def step_attribute(step, name)
|
|
176
|
+
step.public_send(name) if step&.has_attribute?(name)
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def workflow_extras(record, steps)
|
|
180
|
+
step_hashes = steps.map { |step| workflow_step_hash(step) }
|
|
181
|
+
events = WorkflowEvent.where(workflow_id: record.workflow_id).order(id: :desc).limit(EVENT_HISTORY).map do |event|
|
|
182
|
+
{ id: event.id, event: event.event, actor: event.actor, reason: event.reason, step_name: event.step_name,
|
|
183
|
+
attempt: event.attempt, details: event.details, created_at: event.created_at }
|
|
184
|
+
end
|
|
185
|
+
klass = record.workflow_klass
|
|
186
|
+
resume_action, resume_step = WorkflowActionPlan.resume_plan(record, steps.first, klass)
|
|
187
|
+
now = Time.current
|
|
188
|
+
{ arguments: WorkflowPayload.display(record.arguments),
|
|
189
|
+
steps: step_hashes.map { |step| step.merge(durations: WorkflowTimeline.durations(step, now: now)) },
|
|
190
|
+
signals: workflow_signals(record.workflow_id), events: events,
|
|
191
|
+
available_actions: Workflow::States.available_actions(record.status),
|
|
192
|
+
definition_outdated: record.definition_outdated?, class_loaded: !klass.nil?,
|
|
193
|
+
step_names: klass ? klass.workflow_steps.names : [], resume_action: resume_action, resume_step: resume_step,
|
|
194
|
+
signal_names: WorkflowActionPlan.signal_names(steps.first, klass), reruns: workflow_reruns(record.workflow_id),
|
|
195
|
+
timeline: WorkflowTimeline.chart(step_hashes, pause_events(record.workflow_id), status: record.status, now: now) }
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def workflow_step_hash(step)
|
|
199
|
+
{ id: step.id, step_name: step.step_name, attempt: step.attempt, status: step.status, outcome: step.outcome,
|
|
200
|
+
created_at: step.created_at, scheduled_at: step.scheduled_at, enqueued_at: step.enqueued_at,
|
|
201
|
+
started_at: step.started_at, finished_at: step.finished_at, job_id: step.job_id, msg_id: step.msg_id,
|
|
202
|
+
worker: step.worker, app_revision: step.app_revision, error_class: step.error_class,
|
|
203
|
+
error_message: step.error_message, error_backtrace: step.error_backtrace, error_causes: step.error_causes || [],
|
|
204
|
+
output: step.output && WorkflowPayload.display(step.output),
|
|
205
|
+
waiting_since: step_attribute(step, :waiting_since), wait_for: step_attribute(step, :wait_for),
|
|
206
|
+
timeout_at: step_attribute(step, :timeout_at), batch_id: step_attribute(step, :batch_id),
|
|
207
|
+
signal_id: step_attribute(step, :signal_id) }
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def pause_events(workflow_id)
|
|
211
|
+
WorkflowEvent.where(workflow_id: workflow_id, event: WorkflowTimeline::PAUSE_EVENTS).order(:id).limit(PAUSE_HISTORY)
|
|
212
|
+
.map { |event| { event: event.event, created_at: event.created_at } }
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def workflow_signals(workflow_id)
|
|
216
|
+
return [] unless Workflow::Sweep.waiting_ready?
|
|
217
|
+
|
|
218
|
+
WorkflowSignal.where(workflow_id: workflow_id).order(id: :desc).limit(SIGNAL_HISTORY).map do |signal|
|
|
219
|
+
{ id: signal.id, name: signal.name, status: signal.status, idempotency_key: signal.idempotency_key,
|
|
220
|
+
payload: WorkflowPayload.display(signal.read_attribute(:payload)), created_at: signal.created_at,
|
|
221
|
+
claimed_at: signal.claimed_at, consumed_at: signal.consumed_at }
|
|
222
|
+
end
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def workflow_reruns(workflow_id)
|
|
226
|
+
WorkflowEntry.where(rerun_of_workflow_id: workflow_id).order(created_at: :desc).limit(RERUN_LINKS)
|
|
227
|
+
.pluck(:workflow_id, :status, :created_at).map do |id, status, created_at|
|
|
228
|
+
{ workflow_id: id, status: status, created_at: created_at }
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
def workflow_status_count(status)
|
|
233
|
+
return 0 unless Workflow::Sweep.tables_ready?
|
|
234
|
+
|
|
235
|
+
WorkflowEntry.where(status: status).count
|
|
236
|
+
rescue StandardError => e
|
|
237
|
+
Pgbus.logger.error { "[Pgbus::Web] Error counting workflows: #{e.class}: #{e.message}" }
|
|
238
|
+
0
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
def attention_thresholds(config: Pgbus.configuration)
|
|
242
|
+
waiting = Workflow::Sweep.waiting_ready? ? config.workflow_waiting_visibility_threshold : nil
|
|
243
|
+
{ threshold: config.workflow_stall_threshold, waiting_threshold: waiting }
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
end
|
|
247
|
+
end
|
|
248
|
+
end
|
|
@@ -13,6 +13,9 @@ module Pgbus
|
|
|
13
13
|
include ProcessList
|
|
14
14
|
include DashboardSignals
|
|
15
15
|
include DeployTaskList
|
|
16
|
+
include WorkflowList
|
|
17
|
+
include WorkflowActions
|
|
18
|
+
include WorkflowInsights
|
|
16
19
|
|
|
17
20
|
# pgbus_failed_events rows outside the EventBus handler queues; binds
|
|
18
21
|
# handler_queue_names_literal as $1.
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Web
|
|
5
|
+
# What a Workflows dashboard action will do (issue #542), so its button
|
|
6
|
+
# can say so before anyone presses it. Which actions exist at all is
|
|
7
|
+
# Workflow::States.available_actions; this only names their effect.
|
|
8
|
+
module WorkflowActionPlan
|
|
9
|
+
# The status the runs of a needs-attention group are in.
|
|
10
|
+
GROUP_STATUS = { "stalled" => "paused", "paused_error" => "paused", "paused_manual" => "paused",
|
|
11
|
+
"overdue" => "ready", "waiting_overdue" => "waiting" }.freeze
|
|
12
|
+
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
def group_status(reason) = GROUP_STATUS.fetch(reason.to_s)
|
|
16
|
+
|
|
17
|
+
# [action, step_name] for resume! of a paused run, as OutsideVerbs.resume
|
|
18
|
+
# decides it from the run's latest step row:
|
|
19
|
+
# :retry the step failed: it runs again as a new attempt
|
|
20
|
+
# :continue the step completed or was skipped: the next step runs
|
|
21
|
+
# :finish …and it was the last: the run finishes
|
|
22
|
+
# :resend the step was scheduled when the run paused: it is sent again
|
|
23
|
+
# :rewait the step was waiting: it waits again
|
|
24
|
+
# :carry_on the step was still running: the run goes on performing it
|
|
25
|
+
# [nil, nil] when the run is not paused, or its class is gone.
|
|
26
|
+
def resume_plan(record, latest, klass)
|
|
27
|
+
return [nil, nil] unless record.status == "paused" && latest
|
|
28
|
+
|
|
29
|
+
case latest.status
|
|
30
|
+
when "failed" then [:retry, latest.step_name]
|
|
31
|
+
when "scheduled" then [:resend, latest.step_name]
|
|
32
|
+
when "waiting" then [:rewait, latest.step_name]
|
|
33
|
+
when "in_progress" then [:carry_on, latest.step_name]
|
|
34
|
+
when "completed", "skipped" then following_plan(latest, klass)
|
|
35
|
+
else [nil, nil]
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def following_plan(latest, klass)
|
|
40
|
+
return [nil, nil] unless klass
|
|
41
|
+
# resume! raises StepNotDefined for a step the class no longer
|
|
42
|
+
# declares (and no removed_step covers): no plan to promise.
|
|
43
|
+
return [nil, nil] unless klass.workflow_steps.entries.any? { |entry| entry.name == latest.step_name }
|
|
44
|
+
|
|
45
|
+
following = klass.workflow_steps.next_after(latest.step_name)
|
|
46
|
+
following ? [:continue, following.name] : [:finish, nil]
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# The signal names a waiting step takes, for the Send signal select:
|
|
50
|
+
# [name] for a wait_for: name or SignalMatcher, [] for a custom matcher
|
|
51
|
+
# (the form then takes free text) or a step that is not waiting on one.
|
|
52
|
+
def signal_names(latest, klass)
|
|
53
|
+
return [] unless latest&.status == "waiting" && klass
|
|
54
|
+
return [] if latest.has_attribute?(:batch_id) && latest.batch_id
|
|
55
|
+
|
|
56
|
+
matcher = klass.workflow_steps.fetch(latest.step_name)&.wait_for
|
|
57
|
+
name = matcher.respond_to?(:signal_name) ? matcher.signal_name : nil
|
|
58
|
+
name ? [name.to_s] : []
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|