good_pipeline 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Appraisals +25 -0
- data/CHANGELOG.md +58 -0
- data/README.md +67 -11
- data/app/controllers/good_pipeline/frontends_controller.rb +18 -1
- data/app/controllers/good_pipeline/pipelines_controller.rb +254 -36
- data/app/controllers/good_pipeline/themes_controller.rb +15 -0
- data/app/frontend/good_pipeline/dashboard.css +407 -0
- data/app/frontend/good_pipeline/dashboard.js +542 -0
- data/app/helpers/good_pipeline/mermaid_diagram_builder.rb +225 -66
- data/app/helpers/good_pipeline/pipelines_helper.rb +89 -5
- data/app/jobs/good_pipeline/pipeline_callback_job.rb +2 -2
- data/app/jobs/good_pipeline/pipeline_reconciliation_job.rb +1 -2
- data/app/jobs/good_pipeline/step_finished_job.rb +6 -2
- data/app/models/good_pipeline/pipeline_record.rb +7 -4
- data/app/models/good_pipeline/step_record.rb +6 -3
- data/app/views/good_pipeline/pipelines/_actions.html.erb +28 -0
- data/app/views/good_pipeline/pipelines/_chain_links.html.erb +17 -29
- data/app/views/good_pipeline/pipelines/_diagram.html.erb +101 -0
- data/app/views/good_pipeline/pipelines/_filter_bar.html.erb +51 -0
- data/app/views/good_pipeline/pipelines/_kpi_strip.html.erb +74 -0
- data/app/views/good_pipeline/pipelines/_pager.html.erb +31 -0
- data/app/views/good_pipeline/pipelines/_row.html.erb +103 -0
- data/app/views/good_pipeline/pipelines/_row_detail.html.erb +93 -0
- data/app/views/good_pipeline/pipelines/_sidebar.html.erb +115 -0
- data/app/views/good_pipeline/pipelines/_steps_table.html.erb +46 -33
- data/app/views/good_pipeline/pipelines/_table.html.erb +33 -0
- data/app/views/good_pipeline/pipelines/_timeline.html.erb +55 -0
- data/app/views/good_pipeline/pipelines/_topbar.html.erb +54 -0
- data/app/views/good_pipeline/pipelines/definitions.html.erb +88 -55
- data/app/views/good_pipeline/pipelines/index.html.erb +31 -42
- data/app/views/good_pipeline/pipelines/show.html.erb +54 -47
- data/app/views/layouts/good_pipeline/application.html.erb +24 -153
- data/config/routes.rb +4 -0
- data/demo/config/environments/development.rb +3 -0
- data/demo/config/environments/test.rb +2 -1
- data/demo/db/migrate/20260319205325_create_good_jobs.rb +1 -1
- data/demo/db/migrate/20260319205326_create_good_pipeline_tables.rb +1 -1
- data/demo/db/migrate/20260810000000_add_good_pipeline_dashboard_indexes.rb +20 -0
- data/demo/db/seeds.rb +325 -11
- data/demo/docs/screenshots/definitions.png +0 -0
- data/demo/docs/screenshots/show.png +0 -0
- data/demo/test/good_pipeline/test_bulk_enqueue.rb +54 -1
- data/demo/test/good_pipeline/test_cancellation.rb +312 -0
- data/demo/test/good_pipeline/test_cleanup.rb +31 -1
- data/demo/test/good_pipeline/test_coordinator.rb +39 -0
- data/demo/test/good_pipeline/test_dashboard.rb +667 -0
- data/demo/test/good_pipeline/test_dashboard_rerun.rb +244 -0
- data/demo/test/good_pipeline/test_pipeline_callback_job.rb +11 -0
- data/demo/test/good_pipeline/test_pipeline_record.rb +30 -0
- data/demo/test/good_pipeline/test_runner.rb +40 -1
- data/demo/test/good_pipeline/test_step_record.rb +37 -0
- data/demo/test/integration/test_barrier_execution.rb +502 -0
- data/demo/test/integration/test_chain_locking.rb +54 -0
- data/demo/test/integration/test_pipeline_chaining.rb +96 -0
- data/demo/test/test_helper.rb +20 -0
- data/docs/architecture.md +45 -29
- data/docs/branching.md +4 -0
- data/docs/callbacks.md +9 -7
- data/docs/cleanup.md +2 -2
- data/docs/dag-validation.md +16 -0
- data/docs/dashboard.md +84 -22
- data/docs/defining-pipelines.md +34 -1
- data/docs/failure-strategies.md +24 -1
- data/docs/getting-started.md +9 -1
- data/docs/index.md +1 -1
- data/docs/introduction.md +4 -4
- data/docs/monitoring.md +8 -3
- data/docs/pipeline-chaining.md +17 -9
- data/docs/public/screenshots/definitions.png +0 -0
- data/docs/public/screenshots/index.png +0 -0
- data/docs/public/screenshots/show.png +0 -0
- data/docs/screenshots/definitions.png +0 -0
- data/docs/screenshots/index.png +0 -0
- data/docs/screenshots/show.png +0 -0
- data/gemfiles/rails_7_2.gemfile +24 -0
- data/gemfiles/rails_7_2.gemfile.lock +412 -0
- data/gemfiles/rails_8_0.gemfile +24 -0
- data/gemfiles/rails_8_0.gemfile.lock +411 -0
- data/gemfiles/rails_8_1.gemfile +24 -0
- data/gemfiles/rails_8_1.gemfile.lock +415 -0
- data/lib/generators/good_pipeline/install/templates/create_good_pipeline_tables.rb.erb +9 -0
- data/lib/generators/good_pipeline/upgrade/templates/add_good_pipeline_dashboard_indexes.rb.erb +27 -0
- data/lib/generators/good_pipeline/upgrade/upgrade_generator.rb +32 -0
- data/lib/good_pipeline/barrier_compiler.rb +106 -0
- data/lib/good_pipeline/chain_coordinator.rb +9 -5
- data/lib/good_pipeline/constants.rb +2 -0
- data/lib/good_pipeline/coordinator.rb +269 -140
- data/lib/good_pipeline/dashboard/connection_info.rb +67 -0
- data/lib/good_pipeline/dashboard/definition_stages.rb +59 -0
- data/lib/good_pipeline/dashboard/filter_set.rb +77 -0
- data/lib/good_pipeline/dashboard/kpi_calculator.rb +143 -0
- data/lib/good_pipeline/dashboard/sparkline.rb +73 -0
- data/lib/good_pipeline/dashboard/stage_lanes.rb +307 -0
- data/lib/good_pipeline/dashboard/step_timings.rb +50 -0
- data/lib/good_pipeline/dashboard.rb +21 -0
- data/lib/good_pipeline/errors.rb +11 -0
- data/lib/good_pipeline/pipeline.rb +14 -2
- data/lib/good_pipeline/step_definition.rb +16 -1
- data/lib/good_pipeline/version.rb +1 -1
- data/lib/good_pipeline.rb +7 -1
- data/mise.toml +1 -1
- metadata +48 -13
- data/app/frontend/good_pipeline/style.css +0 -597
- data/app/views/good_pipeline/pipelines/_pagination.html.erb +0 -24
- data/app/views/good_pipeline/pipelines/_pipeline_row.html.erb +0 -7
|
@@ -1,106 +1,265 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module GoodPipeline
|
|
4
|
-
class
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
" classDef skipped fill:#bdbdbd,color:#333",
|
|
17
|
-
" classDef skipped_by_branch fill:#bdbdbd,color:#333",
|
|
18
|
-
" classDef halted fill:#8bc34a,color:#fff",
|
|
19
|
-
" classDef branch fill:#ff9800,color:#fff,stroke:#f57c00",
|
|
20
|
-
" classDef terminal fill:#1a1a2e,color:#fff,stroke:#1a1a2e"
|
|
21
|
-
].freeze
|
|
22
|
-
|
|
23
|
-
def initialize(pipeline)
|
|
4
|
+
# Produces only the Mermaid graph body. Theme-dependent class definitions are
|
|
5
|
+
# appended by dashboard.js so changing theme can never reuse stale colours.
|
|
6
|
+
# User-controlled step keys are labels only; generated n0..nN identifiers are
|
|
7
|
+
# used everywhere in the graph syntax.
|
|
8
|
+
class MermaidDiagramBuilder # rubocop:disable Metrics/ClassLength
|
|
9
|
+
MAX_EDGES = 1_000
|
|
10
|
+
MAX_LABEL_LENGTH = 64
|
|
11
|
+
STATUSES = %w[pending running enqueued succeeded failed halted canceled skipped skipped_by_branch].freeze
|
|
12
|
+
Edge = Struct.new(:upstream, :downstream, keyword_init: true)
|
|
13
|
+
private_constant :Edge
|
|
14
|
+
|
|
15
|
+
def initialize(pipeline) # rubocop:disable Metrics/AbcSize
|
|
24
16
|
@pipeline = pipeline
|
|
17
|
+
@steps = ordered_steps(pipeline.steps)
|
|
18
|
+
@dependencies = Array(pipeline.dependencies)
|
|
19
|
+
@step_by_token = @steps.to_h { |step| [step_token(step), step] }
|
|
20
|
+
@node_id_by_token = @steps.each_with_index.to_h { |step, index| [step_token(step), "n#{index}"] }
|
|
21
|
+
@node_id_by_key = @steps.to_h { |step| [step.key.to_s, @node_id_by_token.fetch(step_token(step))] }
|
|
22
|
+
@terminal_node_id = "n#{@steps.length}"
|
|
23
|
+
@edges = dependency_edges
|
|
24
|
+
@edge_count = @dependencies.length
|
|
25
25
|
end
|
|
26
26
|
|
|
27
27
|
def definition_diagram
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
28
|
+
build_diagram do |step|
|
|
29
|
+
next "branch" if branch_step?(step)
|
|
30
|
+
next "barrier" if barrier_step?(step)
|
|
31
|
+
|
|
32
|
+
"step"
|
|
33
|
+
end
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
def status_diagram
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
37
|
+
build_diagram do |step|
|
|
38
|
+
branch_step?(step) ? "branch" : safe_status(step)
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def diagram(mode: :status)
|
|
43
|
+
mode.to_sym == :definition ? definition_diagram : status_diagram
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def overflow? = edge_count > MAX_EDGES
|
|
47
|
+
alias edge_overflow? overflow?
|
|
48
|
+
def renderable? = !overflow?
|
|
49
|
+
|
|
50
|
+
# The number of actual Mermaid arrows includes synthetic terminal and
|
|
51
|
+
# empty-branch-arm edges. `edge_count` intentionally remains the persisted
|
|
52
|
+
# DAG dependency count used by the product's >1000-edge contract.
|
|
53
|
+
def rendered_edge_count
|
|
54
|
+
@rendered_edge_count ||= begin
|
|
55
|
+
terminal_edges = visible_terminal_steps.length
|
|
56
|
+
empty_arm_edges = empty_arm_edge_lines.length
|
|
57
|
+
visible_dependency_edges.length + terminal_edges + empty_arm_edges
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def node_ids
|
|
62
|
+
@node_id_by_key.dup.freeze
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
attr_reader :edge_count, :terminal_node_id
|
|
66
|
+
|
|
67
|
+
def payload(mode: :status)
|
|
68
|
+
{
|
|
69
|
+
graph: diagram(mode: mode),
|
|
70
|
+
edge_count: edge_count,
|
|
71
|
+
overflow: overflow?
|
|
72
|
+
}.freeze
|
|
43
73
|
end
|
|
44
74
|
|
|
45
75
|
private
|
|
46
76
|
|
|
47
|
-
def
|
|
48
|
-
|
|
77
|
+
def build_diagram # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
78
|
+
lines = ["graph TD"]
|
|
79
|
+
@steps.each do |step|
|
|
80
|
+
node_id = node_id_for(step)
|
|
81
|
+
label = escape_label(Dashboard::Topology.label_for(step))
|
|
49
82
|
css_class = yield(step)
|
|
50
|
-
lines << if step
|
|
51
|
-
|
|
83
|
+
lines << if barrier_step?(step)
|
|
84
|
+
%( #{node_id}[["#{label}"]]:::#{css_class})
|
|
85
|
+
elsif branch_step?(step)
|
|
86
|
+
%( #{node_id}{"#{label}"}:::#{css_class})
|
|
52
87
|
else
|
|
53
|
-
|
|
88
|
+
%( #{node_id}("#{label}"):::#{css_class})
|
|
54
89
|
end
|
|
55
90
|
end
|
|
91
|
+
|
|
92
|
+
visible_dependency_edges.each { |edge| lines << dependency_edge_line(edge) }
|
|
93
|
+
lines << %( #{@terminal_node_id}((" ")):::terminal)
|
|
94
|
+
visible_terminal_steps.each { |step| lines << " #{node_id_for(step)} --> #{@terminal_node_id}" }
|
|
95
|
+
lines.concat(empty_arm_edge_lines)
|
|
96
|
+
lines.join("\n")
|
|
56
97
|
end
|
|
57
98
|
|
|
58
|
-
def
|
|
59
|
-
@
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
else
|
|
65
|
-
" #{upstream.key} --> #{downstream.key}"
|
|
66
|
-
end
|
|
99
|
+
def dependency_edges
|
|
100
|
+
@dependencies.filter_map do |dependency|
|
|
101
|
+
upstream_ref, downstream_ref = dependency_endpoints(dependency)
|
|
102
|
+
upstream = resolve_step(upstream_ref)
|
|
103
|
+
downstream = resolve_step(downstream_ref)
|
|
104
|
+
Edge.new(upstream: upstream, downstream: downstream).freeze if upstream && downstream
|
|
67
105
|
end
|
|
68
106
|
end
|
|
69
107
|
|
|
70
|
-
def
|
|
71
|
-
|
|
72
|
-
|
|
108
|
+
def dependency_edge_line(edge)
|
|
109
|
+
from = node_id_for(edge.upstream)
|
|
110
|
+
to = node_id_for(edge.downstream)
|
|
111
|
+
if branch_step?(edge.upstream) && branch_arm_step?(edge.downstream)
|
|
112
|
+
label = escape_edge_label(edge.downstream.branch_arm)
|
|
113
|
+
label.empty? ? " #{from} --> #{to}" : " #{from} -->|#{label}| #{to}"
|
|
114
|
+
else
|
|
115
|
+
" #{from} --> #{to}"
|
|
116
|
+
end
|
|
117
|
+
end
|
|
73
118
|
|
|
74
|
-
|
|
75
|
-
terminal_steps
|
|
119
|
+
def terminal_steps
|
|
120
|
+
@terminal_steps ||= begin
|
|
121
|
+
has_downstream = @edges.to_set { |edge| step_token(edge.upstream) }
|
|
122
|
+
@steps.reject { |step| has_downstream.include?(step_token(step)) }
|
|
123
|
+
end
|
|
124
|
+
end
|
|
76
125
|
|
|
77
|
-
|
|
126
|
+
def visible_dependency_edges
|
|
127
|
+
@visible_dependency_edges ||= @edges.reject { |edge| hidden_dependency_edge?(edge) }
|
|
78
128
|
end
|
|
79
129
|
|
|
80
|
-
def
|
|
81
|
-
|
|
130
|
+
def hidden_dependency_edge?(edge)
|
|
131
|
+
all_empty_branch?(edge.upstream) ||
|
|
132
|
+
(branch_step?(edge.upstream) && barrier_step?(edge.downstream))
|
|
133
|
+
end
|
|
82
134
|
|
|
83
|
-
|
|
84
|
-
|
|
135
|
+
def visible_terminal_steps
|
|
136
|
+
@visible_terminal_steps ||= terminal_steps.reject { |step| all_empty_branch?(step) }
|
|
137
|
+
end
|
|
85
138
|
|
|
86
|
-
|
|
139
|
+
def empty_arm_edge_lines # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
140
|
+
@empty_arm_edge_lines ||= begin
|
|
141
|
+
branch_arm_steps = @steps.select { |step| branch_arm_step?(step) }.group_by do |step|
|
|
142
|
+
step.branch_key.to_s
|
|
143
|
+
end
|
|
87
144
|
|
|
88
|
-
branch_step.
|
|
89
|
-
|
|
90
|
-
targets
|
|
91
|
-
|
|
92
|
-
|
|
145
|
+
@steps.select { |step| branch_step?(step) }.flat_map do |branch_step|
|
|
146
|
+
Array(branch_step.empty_arms).flat_map do |arm_name|
|
|
147
|
+
targets = post_branch_targets(branch_step, branch_arm_steps)
|
|
148
|
+
label = escape_edge_label(arm_name)
|
|
149
|
+
if targets.empty?
|
|
150
|
+
[labelled_edge(node_id_for(branch_step), @terminal_node_id, label)]
|
|
151
|
+
else
|
|
152
|
+
targets.map { |target| labelled_edge(node_id_for(branch_step), node_id_for(target), label) }
|
|
153
|
+
end
|
|
93
154
|
end
|
|
94
155
|
end
|
|
95
156
|
end
|
|
96
157
|
end
|
|
97
158
|
|
|
98
|
-
def
|
|
99
|
-
|
|
159
|
+
def post_branch_targets(branch_step, branch_arm_steps) # rubocop:disable Metrics/AbcSize
|
|
160
|
+
arm_tokens = Array(branch_arm_steps[branch_step.key.to_s]).to_set { |step| step_token(step) }
|
|
161
|
+
return direct_branch_targets(branch_step) if arm_tokens.empty?
|
|
162
|
+
|
|
163
|
+
target_tokens = @edges.filter_map do |edge|
|
|
164
|
+
upstream_token = step_token(edge.upstream)
|
|
165
|
+
downstream_token = step_token(edge.downstream)
|
|
166
|
+
downstream_token if arm_tokens.include?(upstream_token) && !arm_tokens.include?(downstream_token)
|
|
167
|
+
end.to_set
|
|
168
|
+
@steps.select { |step| target_tokens.include?(step_token(step)) }
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def direct_branch_targets(branch_step)
|
|
172
|
+
target_tokens = @edges.filter_map do |edge|
|
|
173
|
+
step_token(edge.downstream) if step_token(edge.upstream) == step_token(branch_step)
|
|
174
|
+
end.to_set
|
|
175
|
+
@steps.select { |step| target_tokens.include?(step_token(step)) }
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def labelled_edge(from, to, label)
|
|
179
|
+
label.empty? ? " #{from} --> #{to}" : " #{from} -->|#{label}| #{to}"
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def dependency_endpoints(dependency) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
183
|
+
if dependency.is_a?(Hash)
|
|
184
|
+
upstream = dependency[:depends_on_step_id] || dependency["depends_on_step_id"] ||
|
|
185
|
+
dependency[:from] || dependency["from"]
|
|
186
|
+
downstream = dependency[:step_id] || dependency["step_id"] ||
|
|
187
|
+
dependency[:to] || dependency["to"]
|
|
188
|
+
return [upstream, downstream]
|
|
189
|
+
end
|
|
190
|
+
return [dependency[0], dependency[1]] if dependency.is_a?(Array)
|
|
191
|
+
|
|
192
|
+
upstream = dependency.depends_on_step_id if dependency.respond_to?(:depends_on_step_id)
|
|
193
|
+
downstream = dependency.step_id if dependency.respond_to?(:step_id)
|
|
194
|
+
upstream ||= dependency.depends_on_step if dependency.respond_to?(:depends_on_step)
|
|
195
|
+
downstream ||= dependency.step if dependency.respond_to?(:step)
|
|
196
|
+
[upstream, downstream]
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def resolve_step(reference)
|
|
200
|
+
return if reference.nil?
|
|
201
|
+
|
|
202
|
+
token = step_token(reference)
|
|
203
|
+
@step_by_token[token] || @steps.find { |step| step.key.to_s == reference.to_s }
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def ordered_steps(steps)
|
|
207
|
+
records = Array(steps)
|
|
208
|
+
return records unless records.all? do |step|
|
|
209
|
+
step.respond_to?(:created_at) && step.respond_to?(:id) && step.created_at && step.id
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
records.sort_by { |step| [step.created_at, step.id.to_s] }
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def step_token(step)
|
|
216
|
+
return unless step
|
|
217
|
+
return step.to_s unless step.respond_to?(:key)
|
|
218
|
+
|
|
219
|
+
id = step.id if step.respond_to?(:id)
|
|
220
|
+
(id.nil? || id.to_s.empty? ? step.key : id).to_s
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def node_id_for(step)
|
|
224
|
+
@node_id_by_token.fetch(step_token(step))
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
def branch_step?(step)
|
|
228
|
+
return step.branch_step? if step.respond_to?(:branch_step?)
|
|
229
|
+
|
|
230
|
+
step.respond_to?(:job_class) && step.job_class.to_s == GoodPipeline::BRANCH_JOB_CLASS
|
|
231
|
+
end
|
|
100
232
|
|
|
101
|
-
|
|
102
|
-
|
|
233
|
+
def branch_arm_step?(step)
|
|
234
|
+
return step.branch_arm_step? if step.respond_to?(:branch_arm_step?)
|
|
235
|
+
|
|
236
|
+
step.respond_to?(:branch_arm) && !step.branch_arm.to_s.empty?
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def barrier_step?(step)
|
|
240
|
+
Dashboard::Topology.barrier_step?(step)
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
def all_empty_branch?(step)
|
|
244
|
+
return false unless branch_step?(step) && Array(step.empty_arms).any?
|
|
245
|
+
|
|
246
|
+
@steps.none? do |candidate|
|
|
247
|
+
branch_arm_step?(candidate) && candidate.branch_key.to_s == step.key.to_s
|
|
103
248
|
end
|
|
104
249
|
end
|
|
250
|
+
|
|
251
|
+
def safe_status(step)
|
|
252
|
+
status = step.respond_to?(:coordination_status) ? step.coordination_status.to_s : "pending"
|
|
253
|
+
STATUSES.include?(status) ? status : "pending"
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
def escape_label(value)
|
|
257
|
+
clean = value.to_s.gsub(/[\u0000-\u001f\u007f]/, "").gsub('"', "#quot;")
|
|
258
|
+
clean.each_char.first(MAX_LABEL_LENGTH).join
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
def escape_edge_label(value)
|
|
262
|
+
escape_label(value).gsub(/[^\w .:-]/, "").each_char.first(MAX_LABEL_LENGTH).join
|
|
263
|
+
end
|
|
105
264
|
end
|
|
106
265
|
end
|
|
@@ -1,11 +1,32 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module GoodPipeline
|
|
4
|
-
module PipelinesHelper
|
|
4
|
+
module PipelinesHelper # rubocop:disable Metrics/ModuleLength
|
|
5
|
+
STATUS_META = {
|
|
6
|
+
"pending" => { label: "pending", glyph: "\u25CB", var: "--st-pending", variable: "--st-pending" },
|
|
7
|
+
"running" => { label: "running", glyph: "\u25CF", var: "--st-running", variable: "--st-running" },
|
|
8
|
+
"enqueued" => { label: "enqueued", glyph: "\u25CF", var: "--st-running", variable: "--st-running" },
|
|
9
|
+
"canceling" => { label: "canceling", glyph: "\u25D0", var: "--st-canceling", variable: "--st-canceling" },
|
|
10
|
+
"canceled" => { label: "canceled", glyph: "\u2298", var: "--st-canceled", variable: "--st-canceled" },
|
|
11
|
+
"succeeded" => { label: "succeeded", glyph: "\u2713", var: "--st-succeeded", variable: "--st-succeeded" },
|
|
12
|
+
"failed" => { label: "failed", glyph: "\u2717", var: "--st-failed", variable: "--st-failed" },
|
|
13
|
+
"halted" => { label: "halted", glyph: "\u2298", var: "--st-halted", variable: "--st-halted" },
|
|
14
|
+
"skipped" => { label: "skipped", glyph: "\u2298", var: "--st-skipped", variable: "--st-skipped" },
|
|
15
|
+
"skipped_by_branch" => {
|
|
16
|
+
label: "skipped\u00B7br", glyph: "\u2298", var: "--st-skipped", variable: "--st-skipped"
|
|
17
|
+
}
|
|
18
|
+
}.transform_values(&:freeze).freeze
|
|
19
|
+
|
|
20
|
+
STATUS_STACK_ORDER = %w[
|
|
21
|
+
succeeded failed halted enqueued running pending canceled skipped skipped_by_branch
|
|
22
|
+
].freeze
|
|
23
|
+
|
|
5
24
|
STATUS_BADGES = {
|
|
6
25
|
"pending" => "\u25CB Pending",
|
|
7
26
|
"running" => "\u25CF Running",
|
|
8
27
|
"enqueued" => "\u25CF Enqueued",
|
|
28
|
+
"canceling" => "\u25D0 Canceling",
|
|
29
|
+
"canceled" => "\u2298 Canceled",
|
|
9
30
|
"succeeded" => "\u2713 Succeeded",
|
|
10
31
|
"failed" => "\u2717 Failed",
|
|
11
32
|
"halted" => "\u2298 Halted",
|
|
@@ -18,16 +39,37 @@ module GoodPipeline
|
|
|
18
39
|
tag.span(label, class: "badge badge-#{status}")
|
|
19
40
|
end
|
|
20
41
|
|
|
42
|
+
def status_pill(status)
|
|
43
|
+
status = status.to_s
|
|
44
|
+
meta = status_meta(status)
|
|
45
|
+
tag.span("#{meta[:glyph]} #{meta[:label]}", class: "gp-status-pill gp-status-#{status}")
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def status_meta(status)
|
|
49
|
+
status = status.to_s
|
|
50
|
+
STATUS_META.fetch(status) do
|
|
51
|
+
{ label: status, glyph: "", var: "--st-pending", variable: "--st-pending" }.freeze
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
21
55
|
def humanized_type(pipeline_type)
|
|
22
56
|
pipeline_type.to_s.safe_constantize&.display_name || pipeline_type.to_s.underscore.titleize
|
|
23
57
|
end
|
|
24
58
|
|
|
59
|
+
def short_type(pipeline_type)
|
|
60
|
+
humanized_type(pipeline_type).sub(/\s*Pipeline\z/, "")
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def step_display_key(step)
|
|
64
|
+
Dashboard::Topology.label_for(step)
|
|
65
|
+
end
|
|
66
|
+
|
|
25
67
|
def relative_time_tag(datetime)
|
|
26
68
|
return "" unless datetime
|
|
27
69
|
|
|
28
70
|
tag.time(relative_time(datetime),
|
|
29
71
|
datetime: datetime.iso8601,
|
|
30
|
-
title: datetime
|
|
72
|
+
title: absolute_time(datetime))
|
|
31
73
|
end
|
|
32
74
|
|
|
33
75
|
def pipeline_duration(pipeline)
|
|
@@ -49,6 +91,14 @@ module GoodPipeline
|
|
|
49
91
|
MermaidDiagramBuilder.new(pipeline).status_diagram
|
|
50
92
|
end
|
|
51
93
|
|
|
94
|
+
def mermaid_edge_count(pipeline)
|
|
95
|
+
MermaidDiagramBuilder.new(pipeline).edge_count
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def mermaid_overflow?(pipeline)
|
|
99
|
+
MermaidDiagramBuilder.new(pipeline).overflow?
|
|
100
|
+
end
|
|
101
|
+
|
|
52
102
|
def good_job_step_url(step)
|
|
53
103
|
return nil unless step.good_job_id
|
|
54
104
|
|
|
@@ -58,18 +108,52 @@ module GoodPipeline
|
|
|
58
108
|
"#{mount_path}/jobs/#{step.good_job_id}"
|
|
59
109
|
end
|
|
60
110
|
|
|
61
|
-
def relative_time(datetime)
|
|
111
|
+
def relative_time(datetime, now: Time.current)
|
|
62
112
|
return "" unless datetime
|
|
63
113
|
|
|
64
|
-
distance = (
|
|
114
|
+
distance = [(now - datetime).to_i, 0].max
|
|
65
115
|
case distance
|
|
66
|
-
when 0..59 then "
|
|
116
|
+
when 0..59 then "#{distance}s ago"
|
|
67
117
|
when 60..3599 then "#{distance / 60}m ago"
|
|
68
118
|
when 3600..86_399 then "#{distance / 3600}h ago"
|
|
69
119
|
else "#{distance / 86_400}d ago"
|
|
70
120
|
end
|
|
71
121
|
end
|
|
72
122
|
|
|
123
|
+
def absolute_time(datetime)
|
|
124
|
+
return "" unless datetime
|
|
125
|
+
|
|
126
|
+
utc_time = datetime.respond_to?(:getutc) ? datetime.getutc : datetime.in_time_zone("UTC")
|
|
127
|
+
utc_time.strftime("%Y-%m-%d %H:%M:%SZ")
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def format_duration(seconds)
|
|
131
|
+
return "\u2014" if seconds.nil?
|
|
132
|
+
|
|
133
|
+
total = [seconds.to_i, 0].max
|
|
134
|
+
return "#{total}s" if total < 60
|
|
135
|
+
|
|
136
|
+
minutes, remaining_seconds = total.divmod(60)
|
|
137
|
+
return "#{minutes}m #{remaining_seconds}s" if total < 3_600
|
|
138
|
+
|
|
139
|
+
hours, remaining_minutes = minutes.divmod(60)
|
|
140
|
+
"#{hours}h #{remaining_minutes}m"
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def format_number(number)
|
|
144
|
+
number.to_i.to_s.reverse.scan(/\d{1,3}/).join(",").reverse
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def step_progress(steps)
|
|
148
|
+
steps = Array(steps)
|
|
149
|
+
[steps.count { |step| step.coordination_status.to_s == "succeeded" }, steps.length]
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def step_progress_title(steps)
|
|
153
|
+
done, total = step_progress(steps)
|
|
154
|
+
"#{done} succeeded of #{total} total steps"
|
|
155
|
+
end
|
|
156
|
+
|
|
73
157
|
def truncated_params(pipeline_params)
|
|
74
158
|
return "" if pipeline_params.blank?
|
|
75
159
|
|
|
@@ -21,8 +21,8 @@ module GoodPipeline
|
|
|
21
21
|
invoke_callback(pipeline, pipeline.on_success_callback, errors)
|
|
22
22
|
when PipelineRecord.statuses[:failed], PipelineRecord.statuses[:halted]
|
|
23
23
|
invoke_callback(pipeline, pipeline.on_failure_callback, errors)
|
|
24
|
-
when PipelineRecord.statuses[:skipped]
|
|
25
|
-
# Skipped pipelines only trigger on_complete (already called above)
|
|
24
|
+
when PipelineRecord.statuses[:skipped], PipelineRecord.statuses[:canceled]
|
|
25
|
+
# Skipped and canceled pipelines only trigger on_complete (already called above)
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
raise_callback_errors(errors) if errors.any?
|
|
@@ -3,8 +3,7 @@
|
|
|
3
3
|
module GoodPipeline
|
|
4
4
|
class PipelineReconciliationJob < ActiveJob::Base
|
|
5
5
|
def perform(batch, _context)
|
|
6
|
-
|
|
7
|
-
GoodPipeline::Coordinator.recompute_pipeline_status(pipeline)
|
|
6
|
+
GoodPipeline::Coordinator.recompute_pipeline_status(batch.properties[:pipeline_id])
|
|
8
7
|
end
|
|
9
8
|
end
|
|
10
9
|
end
|
|
@@ -3,8 +3,12 @@
|
|
|
3
3
|
module GoodPipeline
|
|
4
4
|
class StepFinishedJob < ActiveJob::Base
|
|
5
5
|
def perform(batch, _context)
|
|
6
|
-
|
|
7
|
-
GoodPipeline::Coordinator.complete_step(
|
|
6
|
+
properties = batch.properties
|
|
7
|
+
GoodPipeline::Coordinator.complete_step(
|
|
8
|
+
properties[:step_id],
|
|
9
|
+
pipeline_id: properties[:pipeline_id],
|
|
10
|
+
succeeded: batch.succeeded?
|
|
11
|
+
)
|
|
8
12
|
end
|
|
9
13
|
end
|
|
10
14
|
end
|
|
@@ -8,20 +8,23 @@ module GoodPipeline
|
|
|
8
8
|
self.table_name = "good_pipeline_pipelines"
|
|
9
9
|
self.inheritance_column = nil
|
|
10
10
|
|
|
11
|
-
TERMINAL_STATUSES = %w[succeeded failed halted skipped].freeze
|
|
11
|
+
TERMINAL_STATUSES = %w[succeeded failed halted skipped canceled].freeze
|
|
12
12
|
|
|
13
13
|
VALID_TRANSITIONS = {
|
|
14
|
-
"pending" => %w[running skipped],
|
|
15
|
-
"running" => %w[succeeded failed halted]
|
|
14
|
+
"pending" => %w[running skipped canceled],
|
|
15
|
+
"running" => %w[succeeded failed halted canceling],
|
|
16
|
+
"canceling" => %w[canceled]
|
|
16
17
|
}.freeze
|
|
17
18
|
|
|
18
19
|
enum :status, {
|
|
19
20
|
pending: "pending",
|
|
20
21
|
running: "running",
|
|
22
|
+
canceling: "canceling",
|
|
21
23
|
succeeded: "succeeded",
|
|
22
24
|
failed: "failed",
|
|
23
25
|
halted: "halted",
|
|
24
|
-
skipped: "skipped"
|
|
26
|
+
skipped: "skipped",
|
|
27
|
+
canceled: "canceled"
|
|
25
28
|
}
|
|
26
29
|
|
|
27
30
|
enum :on_failure_strategy, { halt: "halt", continue: "continue", ignore: "ignore" }
|
|
@@ -7,10 +7,10 @@ module GoodPipeline
|
|
|
7
7
|
class StepRecord < ActiveRecord::Base
|
|
8
8
|
self.table_name = "good_pipeline_steps"
|
|
9
9
|
|
|
10
|
-
TERMINAL_COORDINATION_STATUSES = %w[succeeded failed skipped skipped_by_branch halted].freeze
|
|
10
|
+
TERMINAL_COORDINATION_STATUSES = %w[succeeded failed skipped skipped_by_branch halted canceled].freeze
|
|
11
11
|
|
|
12
12
|
VALID_COORDINATION_TRANSITIONS = {
|
|
13
|
-
"pending" => %w[enqueued skipped skipped_by_branch succeeded failed halted],
|
|
13
|
+
"pending" => %w[enqueued skipped skipped_by_branch succeeded failed halted canceled],
|
|
14
14
|
"enqueued" => %w[succeeded failed halted]
|
|
15
15
|
}.freeze
|
|
16
16
|
|
|
@@ -21,7 +21,8 @@ module GoodPipeline
|
|
|
21
21
|
failed: "failed",
|
|
22
22
|
skipped: "skipped",
|
|
23
23
|
skipped_by_branch: "skipped_by_branch",
|
|
24
|
-
halted: "halted"
|
|
24
|
+
halted: "halted",
|
|
25
|
+
canceled: "canceled"
|
|
25
26
|
}
|
|
26
27
|
|
|
27
28
|
enum :on_failure_strategy, { halt: "halt", continue: "continue", ignore: "ignore" }
|
|
@@ -54,6 +55,8 @@ module GoodPipeline
|
|
|
54
55
|
source: :step
|
|
55
56
|
|
|
56
57
|
def branch_step? = job_class == GoodPipeline::BRANCH_JOB_CLASS
|
|
58
|
+
def barrier_step? = job_class == GoodPipeline::BARRIER_JOB_CLASS
|
|
59
|
+
def structural_step? = branch_step? || barrier_step?
|
|
57
60
|
def branch_arm_step? = branch_arm.present?
|
|
58
61
|
|
|
59
62
|
def duration
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
<%
|
|
2
|
+
status = pipeline.status.to_s
|
|
3
|
+
cancel_confirmation = "Cancel this pipeline? Future DAG steps won't be scheduled; enqueued, scheduled, or retrying jobs will continue normally."
|
|
4
|
+
rerun_confirmation = "Start this pipeline again from the beginning as a new standalone execution? It will use the stored parameters and current pipeline definition, may repeat side effects, will not change the original, and will not copy chain relationships."
|
|
5
|
+
%>
|
|
6
|
+
<div class="gp-actions">
|
|
7
|
+
<% if GoodPipeline.dashboard_mutations_enabled? %>
|
|
8
|
+
<% if pipeline.terminal? %>
|
|
9
|
+
<%= button_to "re-run pipeline", rerun_pipeline_path(pipeline), method: :post,
|
|
10
|
+
class: "gp-action",
|
|
11
|
+
form: { class: "gp-action-form", data: { gp_confirm: rerun_confirmation } } %>
|
|
12
|
+
<% else %>
|
|
13
|
+
<button class="gp-action" type="button" disabled title="available after the pipeline finishes">re-run pipeline</button>
|
|
14
|
+
<% end %>
|
|
15
|
+
<% if %w[pending running].include?(status) %>
|
|
16
|
+
<%= button_to "cancel pipeline", cancel_pipeline_path(pipeline), method: :post,
|
|
17
|
+
class: "gp-action gp-action--danger",
|
|
18
|
+
form: { class: "gp-action-form", data: { gp_confirm: cancel_confirmation } } %>
|
|
19
|
+
<% elsif status == "canceling" %>
|
|
20
|
+
<button class="gp-action gp-action--danger" type="button" disabled>cancel requested</button>
|
|
21
|
+
<% elsif status == "canceled" %>
|
|
22
|
+
<button class="gp-action gp-action--danger" type="button" disabled title="pipeline already canceled">cancel pipeline</button>
|
|
23
|
+
<% end %>
|
|
24
|
+
<% end %>
|
|
25
|
+
<% if local_assigns[:detail_link] %>
|
|
26
|
+
<%= link_to "detail page + dag ↗", pipeline_path(pipeline), class: "gp-action gp-action--ghost" %>
|
|
27
|
+
<% end %>
|
|
28
|
+
</div>
|
|
@@ -1,30 +1,18 @@
|
|
|
1
|
-
<%
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
<header><h4>Downstream</h4></header>
|
|
19
|
-
<div class="chain-chips">
|
|
20
|
-
<% pipeline.downstream_pipelines.each do |downstream| %>
|
|
21
|
-
<%= link_to pipeline_path(downstream), class: "chain-chip" do %>
|
|
22
|
-
<span class="chain-label"><%= humanized_type(downstream.type) %></span>
|
|
23
|
-
<%= status_badge(downstream.status) %>
|
|
24
|
-
<% end %>
|
|
25
|
-
<% end %>
|
|
26
|
-
</div>
|
|
27
|
-
</article>
|
|
28
|
-
<% end %>
|
|
29
|
-
</div>
|
|
1
|
+
<% upstream = pipeline.upstream_pipelines.to_a; downstream = pipeline.downstream_pipelines.to_a %>
|
|
2
|
+
<% if upstream.any? || downstream.any? %>
|
|
3
|
+
<section class="gp-card">
|
|
4
|
+
<h3 class="gp-section-heading">chain</h3>
|
|
5
|
+
<div class="gp-chain">
|
|
6
|
+
<% upstream.each do |item| %>
|
|
7
|
+
<%= link_to pipeline_path(item), class: "gp-chain-chip", title: "Upstream execution" do %>
|
|
8
|
+
<span class="gp-chain-arrow">←</span><span><%= respond_to?(:short_type) ? short_type(item.type) : humanized_type(item.type) %></span><span class="gp-status gp-status--<%= item.status %>"><%= item.status %></span>
|
|
9
|
+
<% end %>
|
|
10
|
+
<% end %>
|
|
11
|
+
<% downstream.each do |item| %>
|
|
12
|
+
<%= link_to pipeline_path(item), class: "gp-chain-chip", title: "Downstream execution" do %>
|
|
13
|
+
<span><%= respond_to?(:short_type) ? short_type(item.type) : humanized_type(item.type) %></span><span class="gp-status gp-status--<%= item.status %>"><%= item.status %></span><span class="gp-chain-arrow">→</span>
|
|
14
|
+
<% end %>
|
|
15
|
+
<% end %>
|
|
16
|
+
</div>
|
|
17
|
+
</section>
|
|
30
18
|
<% end %>
|