pgbus 0.19.3 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/README.md +104 -9
- data/Rakefile +6 -1
- data/app/controllers/pgbus/dashboard_controller.rb +2 -0
- data/app/controllers/pgbus/workflow_definitions_controller.rb +28 -0
- data/app/controllers/pgbus/workflows_controller.rb +197 -0
- data/app/frontend/pgbus/modules/charts.js +130 -4
- data/app/frontend/pgbus/modules/charts.min.js +1 -1
- data/app/frontend/pgbus/modules/charts.min.js.map +1 -1
- data/app/frontend/pgbus/style.css +1 -1
- data/app/helpers/pgbus/button_helper.rb +4 -2
- data/app/helpers/pgbus/workflow_insights_helper.rb +129 -0
- data/app/helpers/pgbus/workflows_helper.rb +165 -0
- data/app/models/pgbus/workflow_entry.rb +172 -0
- data/app/models/pgbus/workflow_event.rb +43 -0
- data/app/models/pgbus/workflow_signal.rb +77 -0
- data/app/models/pgbus/workflow_step.rb +142 -0
- data/app/views/layouts/pgbus/application.html.erb +2 -0
- data/app/views/pgbus/dashboard/_stats_cards.html.erb +17 -0
- data/app/views/pgbus/workflows/_actions.html.erb +59 -0
- data/app/views/pgbus/workflows/_events_table.html.erb +24 -0
- data/app/views/pgbus/workflows/_filters.html.erb +29 -0
- data/app/views/pgbus/workflows/_signal_form.html.erb +31 -0
- data/app/views/pgbus/workflows/_signals_table.html.erb +40 -0
- data/app/views/pgbus/workflows/_status_filter.html.erb +12 -0
- data/app/views/pgbus/workflows/_steps_table.html.erb +79 -0
- data/app/views/pgbus/workflows/_timeline.html.erb +39 -0
- data/app/views/pgbus/workflows/_workflows_table.html.erb +59 -0
- data/app/views/pgbus/workflows/attention.html.erb +88 -0
- data/app/views/pgbus/workflows/definitions/_by_revision.html.erb +52 -0
- data/app/views/pgbus/workflows/definitions/_duration_table.html.erb +18 -0
- data/app/views/pgbus/workflows/definitions/_funnel.html.erb +49 -0
- data/app/views/pgbus/workflows/definitions/_pipeline.html.erb +16 -0
- data/app/views/pgbus/workflows/definitions/_step_card.html.erb +31 -0
- data/app/views/pgbus/workflows/definitions/_throughput_table.html.erb +32 -0
- data/app/views/pgbus/workflows/definitions/_window.html.erb +17 -0
- data/app/views/pgbus/workflows/definitions/index.html.erb +71 -0
- data/app/views/pgbus/workflows/definitions/not_found.html.erb +5 -0
- data/app/views/pgbus/workflows/definitions/show.html.erb +80 -0
- data/app/views/pgbus/workflows/index.html.erb +15 -0
- data/app/views/pgbus/workflows/not_found.html.erb +5 -0
- data/app/views/pgbus/workflows/show.html.erb +93 -0
- data/config/i18n-tasks.yml +6 -0
- data/config/locales/da.yml +442 -0
- data/config/locales/de.yml +442 -0
- data/config/locales/en.yml +442 -0
- data/config/locales/es.yml +442 -0
- data/config/locales/fi.yml +442 -0
- data/config/locales/fr.yml +442 -0
- data/config/locales/it.yml +442 -0
- data/config/locales/ja.yml +442 -0
- data/config/locales/nb.yml +442 -0
- data/config/locales/nl.yml +442 -0
- data/config/locales/pt.yml +442 -0
- data/config/locales/sv.yml +442 -0
- data/config/routes.rb +26 -0
- data/lib/generators/pgbus/add_workflow_signals_generator.rb +56 -0
- data/lib/generators/pgbus/add_workflows_generator.rb +54 -0
- data/lib/generators/pgbus/templates/add_workflow_signals.rb.erb +51 -0
- data/lib/generators/pgbus/templates/add_workflows.rb.erb +150 -0
- data/lib/generators/pgbus/templates/migration.rb.erb +127 -0
- data/lib/pgbus/active_job/adapter.rb +67 -26
- data/lib/pgbus/active_job/batch_id.rb +23 -1
- data/lib/pgbus/active_job/delivery.rb +49 -0
- data/lib/pgbus/active_job/executor.rb +3 -1
- data/lib/pgbus/batch/builder.rb +1 -0
- data/lib/pgbus/batch.rb +35 -3
- data/lib/pgbus/configuration/workflow_settings.rb +99 -0
- data/lib/pgbus/configuration.rb +21 -3
- data/lib/pgbus/generators/migration_detector.rb +40 -0
- data/lib/pgbus/instrumentation.rb +32 -0
- data/lib/pgbus/integrations/appsignal/dashboard.json +128 -1
- data/lib/pgbus/integrations/appsignal/event_span.rb +43 -0
- data/lib/pgbus/integrations/appsignal/job_span.rb +47 -0
- data/lib/pgbus/integrations/appsignal/span_subscriber.rb +128 -0
- data/lib/pgbus/integrations/appsignal/subscriber.rb +50 -100
- data/lib/pgbus/mcp/rack_app.rb +7 -2
- data/lib/pgbus/mcp/runner.rb +2 -1
- data/lib/pgbus/mcp/serialization.rb +25 -0
- data/lib/pgbus/mcp/server.rb +53 -10
- data/lib/pgbus/mcp/tools/deploy_tasks_tool.rb +1 -10
- data/lib/pgbus/mcp/tools/workflow_cancel_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_definitions_tool.rb +36 -0
- data/lib/pgbus/mcp/tools/workflow_detail_tool.rb +38 -0
- data/lib/pgbus/mcp/tools/workflow_resume_at_tool.rb +32 -0
- data/lib/pgbus/mcp/tools/workflow_resume_tool.rb +24 -0
- data/lib/pgbus/mcp/tools/workflow_run_now_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_signal_tool.rb +77 -0
- data/lib/pgbus/mcp/tools/workflow_skip_step_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_stats_tool.rb +42 -0
- data/lib/pgbus/mcp/tools/workflows_attention_tool.rb +42 -0
- data/lib/pgbus/mcp/tools/workflows_tool.rb +60 -0
- data/lib/pgbus/mcp/workflow_insights_payload.rb +15 -0
- data/lib/pgbus/mcp/workflow_payloads.rb +53 -0
- data/lib/pgbus/mcp/write_tool.rb +134 -0
- data/lib/pgbus/mcp.rb +15 -0
- data/lib/pgbus/metrics/backends/statsd.rb +6 -0
- data/lib/pgbus/metrics/subscriber.rb +14 -1
- data/lib/pgbus/metrics/workflow_metrics.rb +65 -0
- data/lib/pgbus/process/dispatcher.rb +26 -0
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source/workflow_actions.rb +109 -0
- data/lib/pgbus/web/data_source/workflow_insights.rb +99 -0
- data/lib/pgbus/web/data_source/workflow_list.rb +248 -0
- data/lib/pgbus/web/data_source.rb +3 -0
- data/lib/pgbus/web/workflow_action_plan.rb +62 -0
- data/lib/pgbus/web/workflow_insights_queries.rb +250 -0
- data/lib/pgbus/web/workflow_insights_report.rb +171 -0
- data/lib/pgbus/web/workflow_payload.rb +36 -0
- data/lib/pgbus/web/workflow_query.rb +82 -0
- data/lib/pgbus/web/workflow_state.rb +76 -0
- data/lib/pgbus/web/workflow_timeline.rb +113 -0
- data/lib/pgbus/web/workflow_window.rb +51 -0
- data/lib/pgbus/workflow/batch_finished_job.rb +21 -0
- data/lib/pgbus/workflow/dsl.rb +105 -0
- data/lib/pgbus/workflow/error_capture.rb +63 -0
- data/lib/pgbus/workflow/exception_policy.rb +170 -0
- data/lib/pgbus/workflow/fan_out.rb +157 -0
- data/lib/pgbus/workflow/fan_out_definition.rb +41 -0
- data/lib/pgbus/workflow/flow_control.rb +94 -0
- data/lib/pgbus/workflow/instance.rb +183 -0
- data/lib/pgbus/workflow/outside_verbs.rb +138 -0
- data/lib/pgbus/workflow/payload.rb +53 -0
- data/lib/pgbus/workflow/registry.rb +53 -0
- data/lib/pgbus/workflow/removed_step.rb +14 -0
- data/lib/pgbus/workflow/runner.rb +218 -0
- data/lib/pgbus/workflow/signal_matcher.rb +53 -0
- data/lib/pgbus/workflow/signals.rb +132 -0
- data/lib/pgbus/workflow/states.rb +119 -0
- data/lib/pgbus/workflow/step_context.rb +58 -0
- data/lib/pgbus/workflow/step_definition.rb +131 -0
- data/lib/pgbus/workflow/step_execution.rb +242 -0
- data/lib/pgbus/workflow/step_job.rb +28 -0
- data/lib/pgbus/workflow/step_outcome.rb +151 -0
- data/lib/pgbus/workflow/steps.rb +103 -0
- data/lib/pgbus/workflow/sweep.rb +156 -0
- data/lib/pgbus/workflow/test_helpers.rb +152 -0
- data/lib/pgbus/workflow/wait_timeout.rb +73 -0
- data/lib/pgbus/workflow/waiting.rb +82 -0
- data/lib/pgbus/workflow.rb +183 -0
- data/sig/pgbus/configuration.rbs +22 -0
- metadata +95 -1
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Web
|
|
5
|
+
# The SQL behind the workflow insights (issue #549): one query per block
|
|
6
|
+
# of the page, each bounded by the window and served by an index #540
|
|
7
|
+
# created. The step stats read idx_pgbus_workflow_steps_insights
|
|
8
|
+
# (workflow_class, step_name, finished_at); the run counts read
|
|
9
|
+
# idx_pgbus_workflows_class_status (workflow_class, status, created_at),
|
|
10
|
+
# with every status listed so the created_at range applies under each;
|
|
11
|
+
# the ended runs read idx_pgbus_workflows_status (status, updated_at): a
|
|
12
|
+
# run ends in the UPDATE that sets its finished_at/cancelled_at, so
|
|
13
|
+
# updated_at >= the ended-at bound keeps every row the window wants.
|
|
14
|
+
#
|
|
15
|
+
# Status lists are the States constants, inlined; every value that comes
|
|
16
|
+
# from outside (a class name, a time, a bucket length) is a bind.
|
|
17
|
+
module WorkflowInsightsQueries
|
|
18
|
+
ALL = Workflow::States::WORKFLOW
|
|
19
|
+
ONGOING = Workflow::States::ONGOING
|
|
20
|
+
# Upper bounds (ms) of the end-to-end duration histogram: < 1 s, 10 s,
|
|
21
|
+
# 1 min, 10 min, 1 h, 6 h, 1 d, 7 d, and the open last bin.
|
|
22
|
+
DURATION_BOUNDS_MS = [1_000, 10_000, 60_000, 600_000, 3_600_000, 21_600_000, 86_400_000, 604_800_000].freeze
|
|
23
|
+
REVISIONS = 5
|
|
24
|
+
|
|
25
|
+
module_function
|
|
26
|
+
|
|
27
|
+
# Every workflow_class with a row, by a loose index scan (one probe per
|
|
28
|
+
# class) instead of a DISTINCT over every run.
|
|
29
|
+
def class_names
|
|
30
|
+
sql = <<~SQL
|
|
31
|
+
WITH RECURSIVE classes AS (
|
|
32
|
+
(SELECT workflow_class FROM #{runs} ORDER BY workflow_class LIMIT 1)
|
|
33
|
+
UNION ALL
|
|
34
|
+
SELECT (SELECT w.workflow_class FROM #{runs} w WHERE w.workflow_class > c.workflow_class
|
|
35
|
+
ORDER BY w.workflow_class LIMIT 1)
|
|
36
|
+
FROM classes c WHERE c.workflow_class IS NOT NULL
|
|
37
|
+
)
|
|
38
|
+
SELECT workflow_class FROM classes WHERE workflow_class IS NOT NULL
|
|
39
|
+
SQL
|
|
40
|
+
select(sql, "Classes").rows.map(&:first)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def class_known?(workflow_class)
|
|
44
|
+
select("SELECT 1 FROM #{runs} WHERE workflow_class = $1 LIMIT 1", "Class Known", [workflow_class]).rows.any?
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# The ongoing runs by (class, status, current step).
|
|
48
|
+
def ongoing(workflow_class: nil)
|
|
49
|
+
sql = <<~SQL
|
|
50
|
+
SELECT workflow_class, status, current_step, COUNT(*)
|
|
51
|
+
FROM #{runs}
|
|
52
|
+
WHERE status IN (#{list(ONGOING)})#{" AND workflow_class = $1" if workflow_class}
|
|
53
|
+
GROUP BY workflow_class, status, current_step
|
|
54
|
+
SQL
|
|
55
|
+
select(sql, "Ongoing", [workflow_class].compact).cast_values.map do |klass, status, step, runs|
|
|
56
|
+
{ workflow_class: klass, status: status, step: step, runs: runs.to_i }
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Per class: the runs that finished or were cancelled between +since+
|
|
61
|
+
# and +until_time+, and the end-to-end percentiles (created → finished)
|
|
62
|
+
# of the finished.
|
|
63
|
+
def ended(since:, until_time:, workflow_class: nil)
|
|
64
|
+
sql = <<~SQL
|
|
65
|
+
SELECT workflow_class, COUNT(*) FILTER (WHERE status = 'finished'), COUNT(*) FILTER (WHERE status = 'cancelled'),
|
|
66
|
+
COUNT(e2e_ms), #{percentiles("e2e_ms", 0.5, 0.95, 0.99)}
|
|
67
|
+
FROM (#{ended_runs(until_bind: "$2", class_bind: ("$3" if workflow_class))}) ended
|
|
68
|
+
GROUP BY workflow_class
|
|
69
|
+
SQL
|
|
70
|
+
select(sql, "Ended", [since, until_time, workflow_class].compact).cast_values.map do |klass, finished, cancelled, samples, *pcts|
|
|
71
|
+
{ workflow_class: klass, finished: finished.to_i, cancelled: cancelled.to_i,
|
|
72
|
+
e2e_ms: percentile_hash(%i[p50 p95 p99], pcts, samples) }
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# { bin => runs } of one class's finished runs over DURATION_BOUNDS_MS
|
|
77
|
+
# (bin 0 is under the first bound, the last bin is past the last one).
|
|
78
|
+
def duration_histogram(workflow_class, since:, until_time:)
|
|
79
|
+
sql = <<~SQL
|
|
80
|
+
SELECT width_bucket(e2e_ms::float8, ARRAY[#{DURATION_BOUNDS_MS.join(", ")}]::float8[]), COUNT(*)
|
|
81
|
+
FROM (#{ended_runs(until_bind: "$2", class_bind: "$3")}) ended
|
|
82
|
+
WHERE status = 'finished'
|
|
83
|
+
GROUP BY 1
|
|
84
|
+
SQL
|
|
85
|
+
select(sql, "Durations", [since, until_time, workflow_class]).rows.to_h { |bin, runs| [bin.to_i, runs.to_i] }
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# { bucket start => [started, finished, cancelled] }. Buckets are
|
|
89
|
+
# multiples of +bucket+ seconds on the wall clock of +zone+ (an IANA
|
|
90
|
+
# name), so a day bucket starts at that zone's midnight on both sides
|
|
91
|
+
# of a DST change; a bucket start is that wall clock's epoch seconds
|
|
92
|
+
# (WorkflowInsightsReport.wall_epoch).
|
|
93
|
+
def throughput(workflow_class, since:, until_time:, bucket:, zone: "UTC")
|
|
94
|
+
sql = <<~SQL
|
|
95
|
+
SELECT bucket, SUM(started), SUM(finished), SUM(cancelled) FROM (
|
|
96
|
+
SELECT #{wall_bucket("created_at")} AS bucket, 1 AS started, 0 AS finished, 0 AS cancelled
|
|
97
|
+
FROM #{runs}
|
|
98
|
+
WHERE workflow_class = $2 AND status IN (#{list(ALL)}) AND created_at BETWEEN $1 AND $5
|
|
99
|
+
UNION ALL
|
|
100
|
+
SELECT #{wall_bucket("ended_at")}, 0, (status = 'finished')::int, (status = 'cancelled')::int
|
|
101
|
+
FROM (#{ended_runs(until_bind: "$5", class_bind: "$2")}) ended
|
|
102
|
+
) buckets
|
|
103
|
+
GROUP BY bucket
|
|
104
|
+
SQL
|
|
105
|
+
select(sql, "Throughput", [since, workflow_class, bucket, zone, until_time]).rows.to_h do |start, started, finished, cancelled|
|
|
106
|
+
[start.to_i * bucket, [started.to_i, finished.to_i, cancelled.to_i]]
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# The bucket number of a UTC timestamp column on the wall clock of the
|
|
111
|
+
# zone bound as $4, in buckets of $3 seconds.
|
|
112
|
+
def wall_bucket(column)
|
|
113
|
+
"FLOOR(EXTRACT(EPOCH FROM (#{column} AT TIME ZONE 'UTC' AT TIME ZONE $4)) / $3)::bigint"
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Per step: the attempts that finished between +since+ and +until_time+.
|
|
117
|
+
def step_stats(workflow_class, since:, until_time:)
|
|
118
|
+
sql = <<~SQL
|
|
119
|
+
SELECT step_name, #{stat_columns}, MIN(created_at)
|
|
120
|
+
FROM (#{attempt_rows}) attempts
|
|
121
|
+
GROUP BY step_name
|
|
122
|
+
SQL
|
|
123
|
+
select(sql, "Steps", [workflow_class, since, until_time]).cast_values.to_h do |name, *stats, first_seen_at|
|
|
124
|
+
[name, attempt_stats(stats).merge(first_seen_at: first_seen_at)]
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Per step, over the runs started between +since+ and +until_time+: how many have a row
|
|
129
|
+
# for it (reached) and its mean position in their runs (the step order
|
|
130
|
+
# of a class that no longer loads). Partitioned by the run's bigint id:
|
|
131
|
+
# sorting 50k workflow_id strings under a locale collation costs ~4x.
|
|
132
|
+
def funnel_reached(workflow_class, since:, until_time:)
|
|
133
|
+
sql = <<~SQL
|
|
134
|
+
SELECT step_name, COUNT(*), AVG(position) FROM (
|
|
135
|
+
SELECT s.step_name, RANK() OVER (PARTITION BY w.id ORDER BY MIN(s.id)) AS position
|
|
136
|
+
FROM #{runs} w JOIN #{steps} s ON s.workflow_id = w.workflow_id
|
|
137
|
+
WHERE w.workflow_class = $1 AND w.status IN (#{list(ALL)}) AND w.created_at BETWEEN $2 AND $3
|
|
138
|
+
GROUP BY w.id, s.step_name
|
|
139
|
+
) reached
|
|
140
|
+
GROUP BY step_name
|
|
141
|
+
SQL
|
|
142
|
+
select(sql, "Funnel", [workflow_class, since, until_time]).rows.to_h do |name, reached, position|
|
|
143
|
+
[name, { reached: reached.to_i, position: position.to_f }]
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Where each run started between +since+ and +until_time+ is now:
|
|
148
|
+
# { [step, status] => runs }.
|
|
149
|
+
def funnel_stopped(workflow_class, since:, until_time:)
|
|
150
|
+
sql = <<~SQL
|
|
151
|
+
SELECT current_step, status, COUNT(*)
|
|
152
|
+
FROM #{runs}
|
|
153
|
+
WHERE workflow_class = $1 AND status IN (#{list(ALL)}) AND created_at BETWEEN $2 AND $3
|
|
154
|
+
GROUP BY current_step, status
|
|
155
|
+
SQL
|
|
156
|
+
select(sql, "Stopped", [workflow_class, since, until_time]).rows.to_h { |step, status, runs| [[step, status], runs.to_i] }
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# The step stats of the REVISIONS app revisions that finished an
|
|
160
|
+
# attempt last, per (revision, step) and per revision (step nil).
|
|
161
|
+
def by_revision(workflow_class, since:, until_time:)
|
|
162
|
+
sql = <<~SQL
|
|
163
|
+
WITH revisions AS (
|
|
164
|
+
SELECT app_revision, MIN(finished_at) AS first_seen_at, MAX(finished_at) AS last_seen_at
|
|
165
|
+
FROM #{steps}
|
|
166
|
+
WHERE workflow_class = $1 AND finished_at BETWEEN $2 AND $3 AND app_revision IS NOT NULL
|
|
167
|
+
GROUP BY app_revision
|
|
168
|
+
ORDER BY last_seen_at DESC
|
|
169
|
+
LIMIT #{REVISIONS}
|
|
170
|
+
)
|
|
171
|
+
SELECT attempts.app_revision, revisions.first_seen_at, revisions.last_seen_at, attempts.step_name, #{stat_columns}
|
|
172
|
+
FROM (#{attempt_rows}) attempts
|
|
173
|
+
JOIN revisions ON revisions.app_revision = attempts.app_revision
|
|
174
|
+
GROUP BY GROUPING SETS ((attempts.app_revision, revisions.first_seen_at, revisions.last_seen_at, attempts.step_name),
|
|
175
|
+
(attempts.app_revision, revisions.first_seen_at, revisions.last_seen_at))
|
|
176
|
+
ORDER BY revisions.last_seen_at DESC, attempts.step_name NULLS FIRST
|
|
177
|
+
SQL
|
|
178
|
+
select(sql, "Revisions", [workflow_class, since, until_time]).cast_values.map do |revision, first_seen, last_seen, name, *stats|
|
|
179
|
+
{ revision: revision, first_seen_at: first_seen, last_seen_at: last_seen, step: name, stats: attempt_stats(stats) }
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# The finished and cancelled runs that ended between $1 and +until_bind+
|
|
184
|
+
# (of the class at +class_bind+): never one dated after now (a skewed
|
|
185
|
+
# clock), which the charts would not show either.
|
|
186
|
+
def ended_runs(until_bind:, class_bind: nil)
|
|
187
|
+
<<~SQL
|
|
188
|
+
SELECT workflow_class, status, COALESCE(finished_at, cancelled_at) AS ended_at,
|
|
189
|
+
CASE WHEN status = 'finished' THEN GREATEST(EXTRACT(EPOCH FROM (finished_at - created_at)) * 1000, 0) END AS e2e_ms
|
|
190
|
+
FROM #{runs}
|
|
191
|
+
WHERE status IN ('finished', 'cancelled') AND updated_at >= $1
|
|
192
|
+
AND COALESCE(finished_at, cancelled_at) BETWEEN $1 AND #{until_bind}#{" AND workflow_class = #{class_bind}" if class_bind}
|
|
193
|
+
SQL
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# The attempts of class $1 that finished between $2 and $3 (never one
|
|
197
|
+
# dated after now). run_ms: completed or
|
|
198
|
+
# failed attempts that started; pickup_ms (scheduled → started, never
|
|
199
|
+
# negative): every attempt that started.
|
|
200
|
+
def attempt_rows
|
|
201
|
+
<<~SQL
|
|
202
|
+
SELECT step_name, status, outcome, error_class, app_revision, created_at,
|
|
203
|
+
CASE WHEN status IN ('completed', 'failed') AND started_at IS NOT NULL
|
|
204
|
+
THEN GREATEST(EXTRACT(EPOCH FROM (finished_at - started_at)) * 1000, 0) END AS run_ms,
|
|
205
|
+
CASE WHEN started_at IS NOT NULL
|
|
206
|
+
THEN GREATEST(EXTRACT(EPOCH FROM (started_at - scheduled_at)) * 1000, 0) END AS pickup_ms
|
|
207
|
+
FROM #{steps}
|
|
208
|
+
WHERE workflow_class = $1 AND finished_at BETWEEN $2 AND $3
|
|
209
|
+
SQL
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
STAT_KEYS = %i[attempts completed failed reattempted skipped cancelled].freeze
|
|
213
|
+
|
|
214
|
+
def stat_columns
|
|
215
|
+
<<~SQL.squish
|
|
216
|
+
COUNT(*), COUNT(*) FILTER (WHERE status = 'completed'), COUNT(*) FILTER (WHERE error_class IS NOT NULL),
|
|
217
|
+
COUNT(*) FILTER (WHERE outcome = 'reattempted'), COUNT(*) FILTER (WHERE status = 'skipped'),
|
|
218
|
+
COUNT(*) FILTER (WHERE status = 'cancelled'),
|
|
219
|
+
COUNT(run_ms), #{percentiles("run_ms", 0.5, 0.95, 0.99)},
|
|
220
|
+
COUNT(pickup_ms), #{percentiles("pickup_ms", 0.5, 0.95)}
|
|
221
|
+
SQL
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def attempt_stats(values)
|
|
225
|
+
counts = STAT_KEYS.zip(values.first(STAT_KEYS.size).map(&:to_i)).to_h
|
|
226
|
+
run_samples, run50, run95, run99, pickup_samples, pickup50, pickup95 = values.drop(STAT_KEYS.size)
|
|
227
|
+
counts.merge(run_ms: percentile_hash(%i[p50 p95 p99], [run50, run95, run99], run_samples),
|
|
228
|
+
pickup_ms: percentile_hash(%i[p50 p95], [pickup50, pickup95], pickup_samples))
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def percentiles(column, *fractions)
|
|
232
|
+
fractions.map { |f| "percentile_cont(#{f}) WITHIN GROUP (ORDER BY #{column})" }.join(", ")
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def percentile_hash(keys, values, samples)
|
|
236
|
+
keys.zip(values.map { |v| v&.to_f&.round(1) }).to_h.merge(samples: samples.to_i)
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def list(statuses) = statuses.map { |status| connection.quote(status) }.join(", ")
|
|
240
|
+
|
|
241
|
+
def runs = WorkflowEntry.quoted_table_name
|
|
242
|
+
|
|
243
|
+
def steps = WorkflowStep.quoted_table_name
|
|
244
|
+
|
|
245
|
+
def connection = WorkflowEntry.connection
|
|
246
|
+
|
|
247
|
+
def select(sql, name, binds = []) = connection.exec_query(sql, "WorkflowInsights #{name}", binds)
|
|
248
|
+
end
|
|
249
|
+
end
|
|
250
|
+
end
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Web
|
|
5
|
+
# Shapes WorkflowInsightsQueries' rows into the hashes the insights pages
|
|
6
|
+
# and MCP tools read (issue #549). Pure: no I/O.
|
|
7
|
+
#
|
|
8
|
+
# Steps come in the class's declaration order (gravestones included, as
|
|
9
|
+
# kind "removed"), then any step name the rows hold that the class does
|
|
10
|
+
# not declare (declared: false). For a class that no longer loads, the
|
|
11
|
+
# names in the rows, ordered by their mean position in the runs started
|
|
12
|
+
# in the window. Live counts are keyed by the run's status (ready,
|
|
13
|
+
# performing, waiting, paused), so each links to the filtered list.
|
|
14
|
+
module WorkflowInsightsReport
|
|
15
|
+
LIVE = Workflow::States::ONGOING.map(&:to_sym).freeze
|
|
16
|
+
STOPPED = Workflow::States::WORKFLOW.map(&:to_sym).freeze
|
|
17
|
+
OUTCOMES = %i[completed failed reattempted skipped cancelled].freeze
|
|
18
|
+
# Percentiles over fewer samples than this get a "few samples" note.
|
|
19
|
+
FEW_SAMPLES = 20
|
|
20
|
+
# The real-time step that walks a window's buckets: 15 min, the finest
|
|
21
|
+
# granularity of a UTC offset change.
|
|
22
|
+
WALK = 900
|
|
23
|
+
DAY = 86_400
|
|
24
|
+
|
|
25
|
+
module_function
|
|
26
|
+
|
|
27
|
+
def summary(name, klass, ongoing_rows, ended_row, attention)
|
|
28
|
+
live = ongoing_rows.sum { |row| row[:runs] }
|
|
29
|
+
e2e = ended_row ? ended_row[:e2e_ms].slice(:p50, :p95, :samples) : { p50: nil, p95: nil, samples: 0 }
|
|
30
|
+
{ workflow_class: name, loaded: !klass.nil?, ongoing: live,
|
|
31
|
+
paused: ongoing_rows.select { |row| row[:status] == "paused" }.sum { |row| row[:runs] },
|
|
32
|
+
attention: attention, finished: ended_row&.fetch(:finished) || 0, cancelled: ended_row&.fetch(:cancelled) || 0,
|
|
33
|
+
e2e_ms: e2e, removed_in_flight: removed_in_flight(klass, ongoing_rows) }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def detail(name, klass, window:, since:, now:, ongoing:, ended:, attention:, stats:, histogram:, throughput:, reached:,
|
|
37
|
+
stopped:, revisions:, zone: nil)
|
|
38
|
+
names = step_list(klass, stats, ongoing, reached, stopped)
|
|
39
|
+
summary(name, klass, ongoing, ended, attention).merge(
|
|
40
|
+
window: window.to_h.merge(since: since), bucket_seconds: window.bucket_seconds,
|
|
41
|
+
started: stopped.values.sum, e2e_ms: ended ? ended[:e2e_ms] : empty_percentiles(%i[p50 p95 p99]),
|
|
42
|
+
steps: names.map { |entry| step(entry, stats[entry[:name]], ongoing) },
|
|
43
|
+
duration_histogram: duration_histogram(histogram),
|
|
44
|
+
throughput: throughput_buckets(throughput, since: since, now: now, bucket: window.bucket_seconds, zone: zone),
|
|
45
|
+
funnel: names.map { |entry| funnel_step(entry[:name], reached, stopped) },
|
|
46
|
+
by_revision: by_revision(revisions, names.map { |entry| entry[:name] })
|
|
47
|
+
)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# [{ name:, kind:, removed:, declared: }] in pipeline order.
|
|
51
|
+
def step_list(klass, stats, ongoing, reached, stopped)
|
|
52
|
+
seen = (stats.keys + ongoing.map { |row| row[:step] } + reached.keys + stopped.keys.map(&:first)).compact.uniq
|
|
53
|
+
declared = klass ? klass.workflow_steps.entries.map { |entry| declared_step(entry) } : []
|
|
54
|
+
extra = (seen - declared.map { |entry| entry[:name] }).sort_by do |name|
|
|
55
|
+
[reached.dig(name, :position) || Float::INFINITY, stats.dig(name, :first_seen_at)&.to_f || Float::INFINITY, name]
|
|
56
|
+
end
|
|
57
|
+
declared + extra.map { |name| { name: name, kind: nil, removed: false, declared: false } }
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def declared_step(entry)
|
|
61
|
+
kind = if entry.removed? then "removed"
|
|
62
|
+
elsif entry.kind == "fan_out" then "fan_out"
|
|
63
|
+
elsif entry.waits_for_signal? then "signal"
|
|
64
|
+
else "step"
|
|
65
|
+
end
|
|
66
|
+
{ name: entry.name, kind: kind, removed: entry.removed?, declared: true }
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def step(entry, stats, ongoing)
|
|
70
|
+
counts = LIVE.to_h { |status| [status, 0] }
|
|
71
|
+
ongoing.each { |row| counts[row[:status].to_sym] += row[:runs] if row[:step] == entry[:name] }
|
|
72
|
+
stats ||= { attempts: 0, run_ms: empty_percentiles(%i[p50 p95 p99]), pickup_ms: empty_percentiles(%i[p50 p95]) }
|
|
73
|
+
entry.merge(counts: counts, sample_size: stats[:attempts], outcomes: OUTCOMES.to_h { |key| [key, stats[key].to_i] },
|
|
74
|
+
rates: rates(stats), run_ms: stats[:run_ms], pickup_ms: stats[:pickup_ms])
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Shares of the attempts: failed (an error was recorded), reattempted,
|
|
78
|
+
# skipped. nil without attempts.
|
|
79
|
+
def rates(stats)
|
|
80
|
+
attempts = stats[:attempts].to_i
|
|
81
|
+
%i[failed reattempted skipped].to_h do |key|
|
|
82
|
+
[key, attempts.zero? ? nil : (stats[key].to_f / attempts).round(4)]
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def empty_percentiles(keys) = keys.to_h { |key| [key, nil] }.merge(samples: 0)
|
|
87
|
+
|
|
88
|
+
# Gravestones some ongoing run still sits on: { "reminder" => 3 }.
|
|
89
|
+
def removed_in_flight(klass, ongoing_rows)
|
|
90
|
+
return {} unless klass
|
|
91
|
+
|
|
92
|
+
names = klass.workflow_steps.removed_names
|
|
93
|
+
ongoing_rows.select { |row| names.include?(row[:step]) }
|
|
94
|
+
.each_with_object(Hash.new(0)) { |row, counts| counts[row[:step]] += row[:runs] }.to_h
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def duration_histogram(histogram)
|
|
98
|
+
bounds = WorkflowInsightsQueries::DURATION_BOUNDS_MS
|
|
99
|
+
(0..bounds.size).map do |bin|
|
|
100
|
+
{ from_ms: bin.zero? ? 0 : bounds[bin - 1], to_ms: bounds[bin], count: histogram.fetch(bin, 0) }
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Every bucket of the window, oldest first, empty ones as zeros. +from+
|
|
105
|
+
# and +to+ are the real times its wall-clock start and the next
|
|
106
|
+
# bucket's fall on in +zone+ (a day bucket across a DST change is 23 or
|
|
107
|
+
# 25 hours long; a fall-back hour spans two real hours).
|
|
108
|
+
def throughput_buckets(counts, since:, now:, bucket:, zone: nil)
|
|
109
|
+
zone ||= utc_zone
|
|
110
|
+
bucket_starts(since: since, now: now, bucket: bucket, zone: zone).map do |start|
|
|
111
|
+
started, finished, cancelled = counts.fetch(start, [0, 0, 0])
|
|
112
|
+
{ from: from_wall(start, zone), to: from_wall(start + bucket, zone), started: started, finished: finished,
|
|
113
|
+
cancelled: cancelled }
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# The wall-clock epoch seconds (in +zone+) each bucket from +since+ to
|
|
118
|
+
# +now+ starts at: what WorkflowInsightsQueries.throughput keys by.
|
|
119
|
+
# Walked in real time (steps of at most WALK seconds, finer than any DST
|
|
120
|
+
# change), so a spring-forward night has no bucket for the hour it
|
|
121
|
+
# skips and a fall-back night's repeated hour is one bucket.
|
|
122
|
+
def bucket_starts(since:, now:, bucket:, zone: nil)
|
|
123
|
+
zone ||= utc_zone
|
|
124
|
+
moments = (since.to_i..now.to_i).step([bucket, WALK].min).map { |seconds| Time.at(seconds) } << now
|
|
125
|
+
moments.map { |moment| wall_epoch(moment, zone) / bucket * bucket }.uniq.sort
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# +time+'s wall clock in +zone+, as epoch seconds.
|
|
129
|
+
def wall_epoch(time, zone)
|
|
130
|
+
local = time.in_time_zone(zone)
|
|
131
|
+
local.to_i + local.utc_offset
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# The real (UTC) time a bucket's wall-clock start falls on in +zone+:
|
|
135
|
+
# the first of the two in a repeated hour, or, when a DST change skips
|
|
136
|
+
# the start itself (a midnight that jumps to 01:00), the first moment
|
|
137
|
+
# of the bucket that exists.
|
|
138
|
+
def from_wall(wall, zone)
|
|
139
|
+
(0...DAY).step(WALK) do |later|
|
|
140
|
+
return zone.tzinfo.local_to_utc(Time.at(wall + later).utc, true).utc
|
|
141
|
+
rescue TZInfo::PeriodNotFound
|
|
142
|
+
next
|
|
143
|
+
end
|
|
144
|
+
Time.at(wall).utc
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def utc_zone = ActiveSupport::TimeZone["UTC"]
|
|
148
|
+
|
|
149
|
+
def funnel_step(name, reached, stopped)
|
|
150
|
+
{ name: name, reached: reached.dig(name, :reached) || 0,
|
|
151
|
+
stopped: STOPPED.to_h { |status| [status, stopped.fetch([name, status.to_s], 0)] } }
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# One entry per revision (newest first): its totals and its steps in
|
|
155
|
+
# pipeline order.
|
|
156
|
+
def by_revision(rows, order)
|
|
157
|
+
rows.group_by { |row| row[:revision] }.map do |revision, group|
|
|
158
|
+
total = group.find { |row| row[:step].nil? }
|
|
159
|
+
steps = group.select { |row| row[:step] }.sort_by { |row| order.index(row[:step]) || order.size }
|
|
160
|
+
{ revision: revision, first_seen_at: total[:first_seen_at], last_seen_at: total[:last_seen_at],
|
|
161
|
+
**revision_numbers(total[:stats]),
|
|
162
|
+
steps: steps.map { |row| { name: row[:step], **revision_numbers(row[:stats]) } } }
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def revision_numbers(stats)
|
|
167
|
+
{ attempts: stats[:attempts], rates: rates(stats), run_ms: stats[:run_ms].slice(:p50, :p95, :samples) }
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Web
|
|
5
|
+
# Workflow arguments, step outputs and signal payloads as the Workflows
|
|
6
|
+
# dashboard and its MCP tools show them (issue #542). They are stored
|
|
7
|
+
# through ActiveJob::Arguments; this reads that stored form without
|
|
8
|
+
# deserializing it, so showing a run never loads its records (a GlobalID
|
|
9
|
+
# shows as its URI) and never runs a custom serializer.
|
|
10
|
+
module WorkflowPayload
|
|
11
|
+
BOOKKEEPING = %w[_aj_symbol_keys _aj_ruby2_keywords _aj_hash_with_indifferent_access].freeze
|
|
12
|
+
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
def display(value)
|
|
16
|
+
case value
|
|
17
|
+
when nil then {}
|
|
18
|
+
when Hash then display_hash(value)
|
|
19
|
+
when Array then value.map { |element| element.nil? ? nil : display(element) }
|
|
20
|
+
else value
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def display_hash(hash)
|
|
25
|
+
return hash["_aj_globalid"] if hash.size == 1 && hash.key?("_aj_globalid")
|
|
26
|
+
if hash.key?("_aj_serialized")
|
|
27
|
+
# A serializer that stores one "value" shows it; one that stores
|
|
28
|
+
# several fields (a Range: begin, end, exclude_end) shows those.
|
|
29
|
+
return hash.key?("value") ? display(hash["value"]) : display_hash(hash.except("_aj_serialized"))
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
hash.except(*BOOKKEEPING).transform_values { |nested| nested.nil? ? nil : display(nested) }
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "date"
|
|
4
|
+
require "time"
|
|
5
|
+
require "active_support/core_ext/object/blank"
|
|
6
|
+
require "active_support/core_ext/time"
|
|
7
|
+
|
|
8
|
+
module Pgbus
|
|
9
|
+
module Web
|
|
10
|
+
# The Workflows list's query string (issue #542), checked: a status from
|
|
11
|
+
# DataSource::WorkflowList::STATUSES, the text filters stripped (blank
|
|
12
|
+
# ones dropped, each at most MAX_LENGTH characters) and the created-at
|
|
13
|
+
# and ended-at (issue #549: finished_at or cancelled_at) ranges parsed in
|
|
14
|
+
# Time.zone (a bare date as the start or the end of that day). Anything
|
|
15
|
+
# else raises Invalid, which the controller answers with 400. #to_param
|
|
16
|
+
# is the same query back, for links that keep it.
|
|
17
|
+
class WorkflowQuery
|
|
18
|
+
class Invalid < StandardError; end
|
|
19
|
+
|
|
20
|
+
TEXT_FILTERS = %i[workflow_class step error_class subject_type subject_id workflow_id].freeze
|
|
21
|
+
DATE_FILTERS = %i[created_from created_to ended_from ended_to].freeze
|
|
22
|
+
MAX_LENGTH = 255
|
|
23
|
+
DATE_ONLY = /\A\d{4}-\d{2}-\d{2}\z/
|
|
24
|
+
# A date (what <input type="date"> sends) or an ISO 8601 date and time.
|
|
25
|
+
DATE_OR_TIME = /\A\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?)?\z/
|
|
26
|
+
|
|
27
|
+
attr_reader :status, :filters, :raw
|
|
28
|
+
|
|
29
|
+
def initialize(params)
|
|
30
|
+
@status = (params[:status].presence || "ongoing").to_s
|
|
31
|
+
raise Invalid, "unknown status #{@status.inspect}" unless DataSource::WorkflowList::STATUSES.include?(@status)
|
|
32
|
+
|
|
33
|
+
@raw = {}
|
|
34
|
+
@filters = {}
|
|
35
|
+
TEXT_FILTERS.each { |key| text(key, params[key]) }
|
|
36
|
+
DATE_FILTERS.each { |key| date(key, params[key]) }
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def filtered? = @raw.any?
|
|
40
|
+
|
|
41
|
+
# The query as URL params (status and the filters as given), merged with +extra+.
|
|
42
|
+
def to_param(**extra) = { status: status, **@raw }.merge(extra).compact
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def text(key, value)
|
|
47
|
+
value = scalar(key, value)
|
|
48
|
+
return if value.empty?
|
|
49
|
+
raise Invalid, "#{key} is longer than #{MAX_LENGTH} characters" if value.length > MAX_LENGTH
|
|
50
|
+
|
|
51
|
+
@raw[key] = value
|
|
52
|
+
@filters[key] = value
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def date(key, value)
|
|
56
|
+
value = scalar(key, value)
|
|
57
|
+
return if value.empty?
|
|
58
|
+
|
|
59
|
+
time = parse_time(value) || raise(Invalid, "#{key} #{value.inspect} is not a date")
|
|
60
|
+
time = time.end_of_day if key.end_with?("_to") && value.match?(DATE_ONLY)
|
|
61
|
+
@raw[key] = value
|
|
62
|
+
@filters[key] = time
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def scalar(key, value)
|
|
66
|
+
return "" if value.nil?
|
|
67
|
+
raise Invalid, "#{key} must be a single value" unless value.is_a?(String)
|
|
68
|
+
|
|
69
|
+
value.strip
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Date.valid_date? because Time.zone.parse rolls 2026-02-30 over to March.
|
|
73
|
+
def parse_time(value)
|
|
74
|
+
return nil unless value.match?(DATE_OR_TIME) && Date.valid_date?(*value[0, 10].split("-").map(&:to_i))
|
|
75
|
+
|
|
76
|
+
Time.zone ? Time.zone.parse(value) : Time.parse(value)
|
|
77
|
+
rescue ArgumentError
|
|
78
|
+
nil
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Web
|
|
5
|
+
# The Workflows dashboard's words for one run (issue #542): the badge
|
|
6
|
+
# (its status, or "scheduled" for a ready run whose step waits for a
|
|
7
|
+
# time still ahead) and what it is doing and what happens next, as a
|
|
8
|
+
# pgbus.workflows.next.<key>_html key and its arguments. Arguments named
|
|
9
|
+
# :step, :signal and :batch are names, :ago a past moment and :time a
|
|
10
|
+
# future one; the helper renders each. Pure.
|
|
11
|
+
module WorkflowState
|
|
12
|
+
BADGE_TONES = { "ready" => :gray, "scheduled" => :gray, "performing" => :blue, "waiting" => :yellow,
|
|
13
|
+
"paused" => :yellow, "finished" => :green, "cancelled" => :red }.freeze
|
|
14
|
+
|
|
15
|
+
Result = Data.define(:badge, :badge_tone, :next_key, :next_args)
|
|
16
|
+
|
|
17
|
+
module_function
|
|
18
|
+
|
|
19
|
+
# row – a DataSource#workflows row. threshold – how long a due step may
|
|
20
|
+
# wait for a worker before it counts as overdue (workflow_stall_threshold).
|
|
21
|
+
def present(row, threshold:, now: Time.current)
|
|
22
|
+
badge, key, args = derive(row, threshold, now)
|
|
23
|
+
Result.new(badge: badge, badge_tone: BADGE_TONES.fetch(badge, :gray), next_key: key, next_args: args)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def derive(row, threshold, now)
|
|
27
|
+
step = row[:current_step]
|
|
28
|
+
case row[:status]
|
|
29
|
+
when "performing" then performing(row, step)
|
|
30
|
+
when "waiting" then waiting(row, step)
|
|
31
|
+
when "paused" then paused(row, step)
|
|
32
|
+
when "ready" then ready(row, step, threshold, now)
|
|
33
|
+
when "finished" then ["finished", "finished", { ago: row[:finished_at] || row[:updated_at] }]
|
|
34
|
+
when "cancelled" then ["cancelled", "cancelled", { ago: row[:cancelled_at] || row[:updated_at] }]
|
|
35
|
+
else [row[:status].to_s, "unknown", { status: row[:status].to_s }]
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def performing(row, step)
|
|
40
|
+
attempt = row[:current_attempt].to_i
|
|
41
|
+
return ["performing", "performing", { step: step }] if attempt <= 1
|
|
42
|
+
|
|
43
|
+
["performing", "performing_attempt", { step: step, attempt: attempt }]
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def waiting(row, step)
|
|
47
|
+
if row[:batch_id]
|
|
48
|
+
progress = row[:batch_progress] || {}
|
|
49
|
+
return ["waiting", "waiting_batch", { batch: batch_name(row), done: progress[:done].to_i, total: progress[:total].to_i }]
|
|
50
|
+
end
|
|
51
|
+
return ["waiting", "waiting_step", { step: step }] unless row[:waiting_for]
|
|
52
|
+
|
|
53
|
+
args = { signal: row[:waiting_for], ago: row[:waiting_since] }
|
|
54
|
+
row[:timeout_at] ? ["waiting", "waiting_signal_timeout", args.merge(time: row[:timeout_at])] : ["waiting", "waiting_signal", args]
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def paused(row, step)
|
|
58
|
+
return ["paused", "paused_error", { step: step, error: row[:error_class] }] if row[:error_class]
|
|
59
|
+
|
|
60
|
+
["paused", "paused_manual", { step: step }]
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def ready(row, step, threshold, now)
|
|
64
|
+
due = JobState.time(row[:scheduled_at])
|
|
65
|
+
return ["ready", "ready", { step: step }] unless due
|
|
66
|
+
return ["scheduled", "scheduled", { step: step, time: due }] if due > now
|
|
67
|
+
return ["ready", "overdue", { step: step, ago: due }] if threshold && due < now - threshold
|
|
68
|
+
|
|
69
|
+
["ready", "due", { step: step, ago: due }]
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# The fan-out batch as its description names it: "Class#step".
|
|
73
|
+
def batch_name(row) = "#{row[:workflow_class]}##{row[:current_step]}"
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|