good_pipeline 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Appraisals +25 -0
- data/CHANGELOG.md +58 -0
- data/README.md +67 -11
- data/app/controllers/good_pipeline/frontends_controller.rb +18 -1
- data/app/controllers/good_pipeline/pipelines_controller.rb +254 -36
- data/app/controllers/good_pipeline/themes_controller.rb +15 -0
- data/app/frontend/good_pipeline/dashboard.css +407 -0
- data/app/frontend/good_pipeline/dashboard.js +542 -0
- data/app/helpers/good_pipeline/mermaid_diagram_builder.rb +225 -66
- data/app/helpers/good_pipeline/pipelines_helper.rb +89 -5
- data/app/jobs/good_pipeline/pipeline_callback_job.rb +2 -2
- data/app/jobs/good_pipeline/pipeline_reconciliation_job.rb +1 -2
- data/app/jobs/good_pipeline/step_finished_job.rb +6 -2
- data/app/models/good_pipeline/pipeline_record.rb +7 -4
- data/app/models/good_pipeline/step_record.rb +6 -3
- data/app/views/good_pipeline/pipelines/_actions.html.erb +28 -0
- data/app/views/good_pipeline/pipelines/_chain_links.html.erb +17 -29
- data/app/views/good_pipeline/pipelines/_diagram.html.erb +101 -0
- data/app/views/good_pipeline/pipelines/_filter_bar.html.erb +51 -0
- data/app/views/good_pipeline/pipelines/_kpi_strip.html.erb +74 -0
- data/app/views/good_pipeline/pipelines/_pager.html.erb +31 -0
- data/app/views/good_pipeline/pipelines/_row.html.erb +103 -0
- data/app/views/good_pipeline/pipelines/_row_detail.html.erb +93 -0
- data/app/views/good_pipeline/pipelines/_sidebar.html.erb +115 -0
- data/app/views/good_pipeline/pipelines/_steps_table.html.erb +46 -33
- data/app/views/good_pipeline/pipelines/_table.html.erb +33 -0
- data/app/views/good_pipeline/pipelines/_timeline.html.erb +55 -0
- data/app/views/good_pipeline/pipelines/_topbar.html.erb +54 -0
- data/app/views/good_pipeline/pipelines/definitions.html.erb +88 -55
- data/app/views/good_pipeline/pipelines/index.html.erb +31 -42
- data/app/views/good_pipeline/pipelines/show.html.erb +54 -47
- data/app/views/layouts/good_pipeline/application.html.erb +24 -153
- data/config/routes.rb +4 -0
- data/demo/config/environments/development.rb +3 -0
- data/demo/config/environments/test.rb +2 -1
- data/demo/db/migrate/20260319205325_create_good_jobs.rb +1 -1
- data/demo/db/migrate/20260319205326_create_good_pipeline_tables.rb +1 -1
- data/demo/db/migrate/20260810000000_add_good_pipeline_dashboard_indexes.rb +20 -0
- data/demo/db/seeds.rb +325 -11
- data/demo/docs/screenshots/definitions.png +0 -0
- data/demo/docs/screenshots/show.png +0 -0
- data/demo/test/good_pipeline/test_bulk_enqueue.rb +54 -1
- data/demo/test/good_pipeline/test_cancellation.rb +312 -0
- data/demo/test/good_pipeline/test_cleanup.rb +31 -1
- data/demo/test/good_pipeline/test_coordinator.rb +39 -0
- data/demo/test/good_pipeline/test_dashboard.rb +667 -0
- data/demo/test/good_pipeline/test_dashboard_rerun.rb +244 -0
- data/demo/test/good_pipeline/test_pipeline_callback_job.rb +11 -0
- data/demo/test/good_pipeline/test_pipeline_record.rb +30 -0
- data/demo/test/good_pipeline/test_runner.rb +40 -1
- data/demo/test/good_pipeline/test_step_record.rb +37 -0
- data/demo/test/integration/test_barrier_execution.rb +502 -0
- data/demo/test/integration/test_chain_locking.rb +54 -0
- data/demo/test/integration/test_pipeline_chaining.rb +96 -0
- data/demo/test/test_helper.rb +20 -0
- data/docs/architecture.md +45 -29
- data/docs/branching.md +4 -0
- data/docs/callbacks.md +9 -7
- data/docs/cleanup.md +2 -2
- data/docs/dag-validation.md +16 -0
- data/docs/dashboard.md +84 -22
- data/docs/defining-pipelines.md +34 -1
- data/docs/failure-strategies.md +24 -1
- data/docs/getting-started.md +9 -1
- data/docs/index.md +1 -1
- data/docs/introduction.md +4 -4
- data/docs/monitoring.md +8 -3
- data/docs/pipeline-chaining.md +17 -9
- data/docs/public/screenshots/definitions.png +0 -0
- data/docs/public/screenshots/index.png +0 -0
- data/docs/public/screenshots/show.png +0 -0
- data/docs/screenshots/definitions.png +0 -0
- data/docs/screenshots/index.png +0 -0
- data/docs/screenshots/show.png +0 -0
- data/gemfiles/rails_7_2.gemfile +24 -0
- data/gemfiles/rails_7_2.gemfile.lock +412 -0
- data/gemfiles/rails_8_0.gemfile +24 -0
- data/gemfiles/rails_8_0.gemfile.lock +411 -0
- data/gemfiles/rails_8_1.gemfile +24 -0
- data/gemfiles/rails_8_1.gemfile.lock +415 -0
- data/lib/generators/good_pipeline/install/templates/create_good_pipeline_tables.rb.erb +9 -0
- data/lib/generators/good_pipeline/upgrade/templates/add_good_pipeline_dashboard_indexes.rb.erb +27 -0
- data/lib/generators/good_pipeline/upgrade/upgrade_generator.rb +32 -0
- data/lib/good_pipeline/barrier_compiler.rb +106 -0
- data/lib/good_pipeline/chain_coordinator.rb +9 -5
- data/lib/good_pipeline/constants.rb +2 -0
- data/lib/good_pipeline/coordinator.rb +269 -140
- data/lib/good_pipeline/dashboard/connection_info.rb +67 -0
- data/lib/good_pipeline/dashboard/definition_stages.rb +59 -0
- data/lib/good_pipeline/dashboard/filter_set.rb +77 -0
- data/lib/good_pipeline/dashboard/kpi_calculator.rb +143 -0
- data/lib/good_pipeline/dashboard/sparkline.rb +73 -0
- data/lib/good_pipeline/dashboard/stage_lanes.rb +307 -0
- data/lib/good_pipeline/dashboard/step_timings.rb +50 -0
- data/lib/good_pipeline/dashboard.rb +21 -0
- data/lib/good_pipeline/errors.rb +11 -0
- data/lib/good_pipeline/pipeline.rb +14 -2
- data/lib/good_pipeline/step_definition.rb +16 -1
- data/lib/good_pipeline/version.rb +1 -1
- data/lib/good_pipeline.rb +7 -1
- data/mise.toml +1 -1
- metadata +48 -13
- data/app/frontend/good_pipeline/style.css +0 -597
- data/app/views/good_pipeline/pipelines/_pagination.html.erb +0 -24
- data/app/views/good_pipeline/pipelines/_pipeline_row.html.erb +0 -7
|
@@ -1,155 +1,26 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
<
|
|
4
|
-
<
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
<
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
<
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
<main class="dashboard-content">
|
|
1
|
+
<% theme = %w[dark light].include?(cookies[:good_pipeline_theme]) ? cookies[:good_pipeline_theme] : "dark" %>
|
|
2
|
+
<!doctype html>
|
|
3
|
+
<html lang="en" data-gp-theme="<%= theme %>">
|
|
4
|
+
<head>
|
|
5
|
+
<meta charset="utf-8">
|
|
6
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
7
|
+
<meta name="csrf-token" content="<%= form_authenticity_token %>">
|
|
8
|
+
<meta name="good-pipeline-theme-url" content="<%= theme_path %>">
|
|
9
|
+
<meta name="turbo-visit-control" content="reload">
|
|
10
|
+
<title>GoodPipeline</title>
|
|
11
|
+
<link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'%3E%3Crect width='32' height='32' rx='6' fill='%23111317'/%3E%3Ctext x='5' y='22' fill='%23e6e9ef' font-family='monospace' font-size='16'%3Egp%3C/text%3E%3C/svg%3E">
|
|
12
|
+
|
|
13
|
+
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
14
|
+
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
15
|
+
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">
|
|
16
|
+
<link rel="stylesheet" href="<%= frontend_static_path(id: "dashboard", format: :css) %>">
|
|
17
|
+
<script src="https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.min.js" defer nonce="<%= content_security_policy_nonce %>"></script>
|
|
18
|
+
<script src="<%= frontend_static_path(id: "dashboard", format: :js) %>" defer nonce="<%= content_security_policy_nonce %>"></script>
|
|
19
|
+
</head>
|
|
20
|
+
<body>
|
|
21
|
+
<% if flash[:notice] %><div class="gp-flash gp-flash--notice" role="status"><%= flash[:notice] %></div><% end %>
|
|
22
|
+
<% if flash[:alert] %><div class="gp-flash gp-flash--alert" role="alert"><%= flash[:alert] %></div><% end %>
|
|
24
23
|
<%= yield %>
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
<script>
|
|
28
|
-
mermaid.initialize({
|
|
29
|
-
startOnLoad: true,
|
|
30
|
-
theme: 'neutral',
|
|
31
|
-
flowchart: { curve: 'basis', nodeSpacing: 30, rankSpacing: 50 }
|
|
32
|
-
});
|
|
33
|
-
document.querySelectorAll('tr[data-href]').forEach(function(row) {
|
|
34
|
-
row.addEventListener('click', function() {
|
|
35
|
-
window.location = this.dataset.href;
|
|
36
|
-
});
|
|
37
|
-
});
|
|
38
|
-
</script>
|
|
39
|
-
<script>
|
|
40
|
-
(function() {
|
|
41
|
-
var MIN_SCALE = 0.5, MAX_SCALE = 4, ZOOM_STEP = 0.25;
|
|
42
|
-
var states = new WeakMap();
|
|
43
|
-
|
|
44
|
-
function getState(container) {
|
|
45
|
-
if (!states.has(container)) states.set(container, { scale: 1, panX: 0, panY: 0 });
|
|
46
|
-
return states.get(container);
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
function applyTransform(container) {
|
|
50
|
-
var s = getState(container);
|
|
51
|
-
var canvas = container.querySelector('.diagram-canvas');
|
|
52
|
-
if (canvas) canvas.style.transform = 'translate(' + s.panX + 'px, ' + s.panY + 'px) scale(' + s.scale + ')';
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
function clampScale(val) {
|
|
56
|
-
return Math.min(MAX_SCALE, Math.max(MIN_SCALE, val));
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
function zoomAroundPoint(s, centerX, centerY, newScale) {
|
|
60
|
-
var oldScale = s.scale;
|
|
61
|
-
newScale = clampScale(newScale);
|
|
62
|
-
if (newScale === oldScale) return;
|
|
63
|
-
var ratio = newScale / oldScale;
|
|
64
|
-
s.panX = centerX - (centerX - s.panX) * ratio;
|
|
65
|
-
s.panY = centerY - (centerY - s.panY) * ratio;
|
|
66
|
-
s.scale = newScale;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
// Button actions via event delegation
|
|
70
|
-
document.addEventListener('click', function(e) {
|
|
71
|
-
var btn = e.target.closest('[data-action]');
|
|
72
|
-
if (!btn) return;
|
|
73
|
-
var container = btn.closest('.diagram-container');
|
|
74
|
-
if (!container) return;
|
|
75
|
-
var s = getState(container);
|
|
76
|
-
var action = btn.dataset.action;
|
|
77
|
-
|
|
78
|
-
if (action === 'zoom-in' || action === 'zoom-out') {
|
|
79
|
-
var viewport = container.querySelector('.diagram-viewport');
|
|
80
|
-
var rect = viewport.getBoundingClientRect();
|
|
81
|
-
var centerX = rect.width / 2;
|
|
82
|
-
var centerY = rect.height / 2;
|
|
83
|
-
var delta = action === 'zoom-in' ? ZOOM_STEP : -ZOOM_STEP;
|
|
84
|
-
zoomAroundPoint(s, centerX, centerY, s.scale + delta);
|
|
85
|
-
} else if (action === 'zoom-reset') {
|
|
86
|
-
s.scale = 1; s.panX = 0; s.panY = 0;
|
|
87
|
-
} else if (action === 'fullscreen') {
|
|
88
|
-
if (document.fullscreenElement === container) {
|
|
89
|
-
document.exitFullscreen();
|
|
90
|
-
} else {
|
|
91
|
-
container.requestFullscreen();
|
|
92
|
-
}
|
|
93
|
-
return;
|
|
94
|
-
}
|
|
95
|
-
applyTransform(container);
|
|
96
|
-
});
|
|
97
|
-
|
|
98
|
-
// Mouse wheel zoom toward cursor
|
|
99
|
-
document.querySelectorAll('.diagram-viewport').forEach(function(viewport) {
|
|
100
|
-
viewport.addEventListener('wheel', function(e) {
|
|
101
|
-
e.preventDefault();
|
|
102
|
-
var container = viewport.closest('.diagram-container');
|
|
103
|
-
var s = getState(container);
|
|
104
|
-
var rect = viewport.getBoundingClientRect();
|
|
105
|
-
var cursorX = e.clientX - rect.left;
|
|
106
|
-
var cursorY = e.clientY - rect.top;
|
|
107
|
-
var delta = e.deltaY < 0 ? ZOOM_STEP : -ZOOM_STEP;
|
|
108
|
-
zoomAroundPoint(s, cursorX, cursorY, s.scale + delta);
|
|
109
|
-
applyTransform(container);
|
|
110
|
-
}, { passive: false });
|
|
111
|
-
});
|
|
112
|
-
|
|
113
|
-
// Click-drag pan
|
|
114
|
-
document.querySelectorAll('.diagram-viewport').forEach(function(viewport) {
|
|
115
|
-
viewport.addEventListener('mousedown', function(e) {
|
|
116
|
-
if (e.button !== 0) return;
|
|
117
|
-
var container = viewport.closest('.diagram-container');
|
|
118
|
-
var canvas = container.querySelector('.diagram-canvas');
|
|
119
|
-
var s = getState(container);
|
|
120
|
-
var startX = e.clientX - s.panX;
|
|
121
|
-
var startY = e.clientY - s.panY;
|
|
122
|
-
canvas.classList.add('dragging');
|
|
123
|
-
|
|
124
|
-
function onMove(ev) {
|
|
125
|
-
s.panX = ev.clientX - startX;
|
|
126
|
-
s.panY = ev.clientY - startY;
|
|
127
|
-
applyTransform(container);
|
|
128
|
-
}
|
|
129
|
-
function onUp() {
|
|
130
|
-
canvas.classList.remove('dragging');
|
|
131
|
-
document.removeEventListener('mousemove', onMove);
|
|
132
|
-
document.removeEventListener('mouseup', onUp);
|
|
133
|
-
}
|
|
134
|
-
document.addEventListener('mousemove', onMove);
|
|
135
|
-
document.addEventListener('mouseup', onUp);
|
|
136
|
-
});
|
|
137
|
-
});
|
|
138
|
-
|
|
139
|
-
// Reset zoom when switching definitions
|
|
140
|
-
document.querySelectorAll('.definition-item').forEach(function(btn) {
|
|
141
|
-
btn.addEventListener('click', function() {
|
|
142
|
-
var type = btn.dataset.pipeline;
|
|
143
|
-
var panel = document.querySelector('.definition-panel[data-pipeline="' + type + '"]');
|
|
144
|
-
if (!panel) return;
|
|
145
|
-
var container = panel.querySelector('.diagram-container');
|
|
146
|
-
if (!container) return;
|
|
147
|
-
var s = getState(container);
|
|
148
|
-
s.scale = 1; s.panX = 0; s.panY = 0;
|
|
149
|
-
applyTransform(container);
|
|
150
|
-
});
|
|
151
|
-
});
|
|
152
|
-
})();
|
|
153
|
-
</script>
|
|
154
|
-
</body>
|
|
24
|
+
<div id="gp-aria-live" class="gp-sr-only" aria-live="polite" aria-atomic="true"></div>
|
|
25
|
+
</body>
|
|
155
26
|
</html>
|
data/config/routes.rb
CHANGED
|
@@ -5,8 +5,12 @@ GoodPipeline::Engine.routes.draw do
|
|
|
5
5
|
|
|
6
6
|
resources :pipelines, only: %i[index show] do
|
|
7
7
|
get :definitions, on: :collection
|
|
8
|
+
post :cancel, on: :member
|
|
9
|
+
post :rerun, on: :member
|
|
8
10
|
end
|
|
9
11
|
|
|
12
|
+
patch :theme, to: "themes#update", as: :theme
|
|
13
|
+
|
|
10
14
|
scope :frontend, controller: :frontends, defaults: { version: GoodPipeline::VERSION.tr(".", "-") } do
|
|
11
15
|
get "static/:version/:id", action: :static, as: :frontend_static
|
|
12
16
|
end
|
|
@@ -4,7 +4,8 @@ Rails.application.configure do
|
|
|
4
4
|
config.cache_classes = true
|
|
5
5
|
config.eager_load = false
|
|
6
6
|
config.consider_all_requests_local = true
|
|
7
|
-
config.action_controller.perform_caching =
|
|
7
|
+
config.action_controller.perform_caching = true if config.respond_to?(:action_controller)
|
|
8
|
+
config.cache_store = :memory_store
|
|
8
9
|
config.active_support.deprecation = :stderr
|
|
9
10
|
config.good_job.execution_mode = :external
|
|
10
11
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
class CreateGoodPipelineTables < ActiveRecord::Migration[
|
|
3
|
+
class CreateGoodPipelineTables < ActiveRecord::Migration[7.2]
|
|
4
4
|
def change
|
|
5
5
|
# Uncomment for Postgres v12 or earlier to enable gen_random_uuid() support
|
|
6
6
|
# enable_extension 'pgcrypto'
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class AddGoodPipelineDashboardIndexes < ActiveRecord::Migration[7.2]
|
|
4
|
+
disable_ddl_transaction!
|
|
5
|
+
|
|
6
|
+
def change
|
|
7
|
+
add_index :good_pipeline_pipelines, %i[type created_at id],
|
|
8
|
+
order: { created_at: :desc, id: :desc },
|
|
9
|
+
name: :index_gp_pipelines_on_type_created_at_id,
|
|
10
|
+
algorithm: :concurrently, if_not_exists: true
|
|
11
|
+
add_index :good_pipeline_pipelines, %i[status created_at id],
|
|
12
|
+
order: { created_at: :desc, id: :desc },
|
|
13
|
+
name: :index_gp_pipelines_on_status_created_at_id,
|
|
14
|
+
algorithm: :concurrently, if_not_exists: true
|
|
15
|
+
add_index :good_pipeline_pipelines, %i[created_at id],
|
|
16
|
+
order: { created_at: :desc, id: :desc },
|
|
17
|
+
name: :index_gp_pipelines_on_created_at_id,
|
|
18
|
+
algorithm: :concurrently, if_not_exists: true
|
|
19
|
+
end
|
|
20
|
+
end
|
data/demo/db/seeds.rb
CHANGED
|
@@ -1,36 +1,115 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "securerandom"
|
|
4
|
+
|
|
3
5
|
puts "Seeding GoodPipeline demo data..."
|
|
4
6
|
|
|
7
|
+
SEED_JOB_QUEUE = "good_pipeline_demo_seed"
|
|
8
|
+
SEED_FUTURE = Time.utc(2100, 1, 1)
|
|
9
|
+
NOW = Time.current.change(usec: 0)
|
|
10
|
+
|
|
11
|
+
# GoodJob rows have no foreign key from GoodPipeline, so tag synthetic rows by
|
|
12
|
+
# queue name and remove them explicitly before rebuilding the demo fixtures.
|
|
13
|
+
GoodJob::Job.where(queue_name: SEED_JOB_QUEUE).delete_all
|
|
5
14
|
GoodPipeline::ChainRecord.delete_all
|
|
6
15
|
GoodPipeline::DependencyRecord.delete_all
|
|
7
16
|
GoodPipeline::StepRecord.delete_all
|
|
8
17
|
GoodPipeline::PipelineRecord.delete_all
|
|
9
18
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
19
|
+
def create_pipeline(type:, status:, params:, strategy: "halt", age: nil, created_at: nil, duration: nil,
|
|
20
|
+
halt_triggered: false)
|
|
21
|
+
timestamp = created_at || (NOW - age)
|
|
13
22
|
pipeline = GoodPipeline::PipelineRecord.create!(
|
|
14
23
|
type: type, status: status, params: params,
|
|
15
24
|
on_failure_strategy: strategy, halt_triggered: halt_triggered
|
|
16
25
|
)
|
|
17
26
|
pipeline.update_columns(
|
|
18
|
-
created_at:
|
|
19
|
-
updated_at: duration ?
|
|
27
|
+
created_at: timestamp,
|
|
28
|
+
updated_at: duration ? timestamp + duration : timestamp
|
|
20
29
|
)
|
|
21
30
|
pipeline
|
|
22
31
|
end
|
|
23
32
|
|
|
33
|
+
def active_job_payload(job_id:, job_class:, scheduled_at:, enqueued_at:)
|
|
34
|
+
{
|
|
35
|
+
"job_class" => job_class,
|
|
36
|
+
"job_id" => job_id,
|
|
37
|
+
"provider_job_id" => job_id,
|
|
38
|
+
"queue_name" => SEED_JOB_QUEUE,
|
|
39
|
+
"priority" => 0,
|
|
40
|
+
"arguments" => [],
|
|
41
|
+
"executions" => 0,
|
|
42
|
+
"exception_executions" => {},
|
|
43
|
+
"locale" => "en",
|
|
44
|
+
"timezone" => "UTC",
|
|
45
|
+
"enqueued_at" => enqueued_at.utc.iso8601(9),
|
|
46
|
+
"scheduled_at" => scheduled_at.utc.iso8601(9)
|
|
47
|
+
}
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def create_seed_job(job_class:, state:, created_at:, duration:, error: nil)
|
|
51
|
+
job_id = SecureRandom.uuid
|
|
52
|
+
unfinished = %i[running queued].include?(state)
|
|
53
|
+
scheduled_at = unfinished ? SEED_FUTURE : created_at
|
|
54
|
+
performed_at = %i[completed running].include?(state) ? created_at : nil
|
|
55
|
+
finished_at = state == :completed ? created_at + duration : nil
|
|
56
|
+
|
|
57
|
+
GoodJob::Job.create!(
|
|
58
|
+
id: job_id,
|
|
59
|
+
active_job_id: job_id,
|
|
60
|
+
queue_name: SEED_JOB_QUEUE,
|
|
61
|
+
priority: 0,
|
|
62
|
+
serialized_params: active_job_payload(
|
|
63
|
+
job_id: job_id,
|
|
64
|
+
job_class: job_class,
|
|
65
|
+
scheduled_at: scheduled_at,
|
|
66
|
+
enqueued_at: created_at
|
|
67
|
+
),
|
|
68
|
+
scheduled_at: scheduled_at,
|
|
69
|
+
performed_at: performed_at,
|
|
70
|
+
finished_at: finished_at,
|
|
71
|
+
error: error,
|
|
72
|
+
executions_count: state == :queued ? 0 : 1,
|
|
73
|
+
job_class: job_class,
|
|
74
|
+
created_at: created_at,
|
|
75
|
+
updated_at: finished_at || created_at
|
|
76
|
+
)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def inferred_job_state(status)
|
|
80
|
+
case status
|
|
81
|
+
when "succeeded", "failed", "halted" then :completed
|
|
82
|
+
when "enqueued" then :queued
|
|
83
|
+
else :none
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
24
87
|
def add_steps(pipeline, *step_defs)
|
|
25
88
|
records = {}
|
|
26
|
-
step_defs.
|
|
89
|
+
step_defs.each_with_index do |definition, index|
|
|
27
90
|
branch_hash = definition.slice(:decides, :branch_result, :empty_arms, :branch_key, :branch_arm).transform_keys(&:to_s)
|
|
91
|
+
status = definition.fetch(:status, "succeeded")
|
|
92
|
+
job_state = definition.fetch(:job_state, inferred_job_state(status))
|
|
93
|
+
started_at = pipeline.created_at + definition.fetch(:start_after, (index + 1) * 5).seconds
|
|
94
|
+
job_duration = definition.fetch(:duration, 10).seconds
|
|
95
|
+
error = if status == "failed"
|
|
96
|
+
"#{definition.fetch(:error_class, 'RuntimeError')}: #{definition.fetch(:error_message, 'Synthetic failure')}"
|
|
97
|
+
end
|
|
98
|
+
good_job = if %i[completed running queued].include?(job_state)
|
|
99
|
+
create_seed_job(
|
|
100
|
+
job_class: definition[:job], state: job_state, created_at: started_at,
|
|
101
|
+
duration: job_duration, error: error
|
|
102
|
+
)
|
|
103
|
+
end
|
|
104
|
+
good_job_id = job_state == :dangling ? SecureRandom.uuid : good_job&.id
|
|
28
105
|
|
|
29
106
|
records[definition[:key]] = GoodPipeline::StepRecord.create!(
|
|
30
107
|
pipeline: pipeline, key: definition[:key], job_class: definition[:job],
|
|
31
|
-
coordination_status:
|
|
108
|
+
coordination_status: status, good_job_id: good_job_id,
|
|
32
109
|
error_class: definition[:error_class], error_message: definition[:error_message],
|
|
33
|
-
branch: branch_hash
|
|
110
|
+
branch: branch_hash,
|
|
111
|
+
created_at: pipeline.created_at + index.fdiv(1_000_000),
|
|
112
|
+
updated_at: pipeline.created_at + index.fdiv(1_000_000)
|
|
34
113
|
)
|
|
35
114
|
end
|
|
36
115
|
records
|
|
@@ -40,6 +119,63 @@ def add_edges(pipeline, steps, *edges)
|
|
|
40
119
|
edges.each do |from, to|
|
|
41
120
|
GoodPipeline::DependencyRecord.create!(pipeline: pipeline, step: steps[to], depends_on_step: steps[from])
|
|
42
121
|
end
|
|
122
|
+
normalize_job_timings(pipeline)
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def normalize_job_timings(pipeline)
|
|
126
|
+
ordered_steps = pipeline.steps.order(:created_at, :id).to_a
|
|
127
|
+
dependencies = pipeline.dependencies.to_a
|
|
128
|
+
jobs = GoodJob::Job.where(id: ordered_steps.filter_map(&:good_job_id)).index_by(&:id)
|
|
129
|
+
parents = dependencies.group_by(&:step_id).transform_values do |records|
|
|
130
|
+
records.map(&:depends_on_step_id)
|
|
131
|
+
end
|
|
132
|
+
by_id = ordered_steps.index_by(&:id)
|
|
133
|
+
remaining = ordered_steps.to_h { |step| [step.id, parents.fetch(step.id, []).length] }
|
|
134
|
+
children = dependencies.group_by(&:depends_on_step_id)
|
|
135
|
+
ready = ordered_steps.select { |step| remaining.fetch(step.id).zero? }
|
|
136
|
+
|
|
137
|
+
until ready.empty?
|
|
138
|
+
step = ready.shift
|
|
139
|
+
job = jobs[step.good_job_id]
|
|
140
|
+
upstream_finishes = parents.fetch(step.id, []).filter_map do |id|
|
|
141
|
+
upstream = by_id[id]
|
|
142
|
+
jobs[upstream&.good_job_id]&.finished_at
|
|
143
|
+
end
|
|
144
|
+
shift_seed_job_after(job, upstream_finishes.max) if job&.performed_at && upstream_finishes.any?
|
|
145
|
+
|
|
146
|
+
children.fetch(step.id, []).each do |dependency|
|
|
147
|
+
remaining[dependency.step_id] -= 1
|
|
148
|
+
ready << by_id.fetch(dependency.step_id) if remaining.fetch(dependency.step_id).zero?
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def shift_seed_job_after(job, earliest_start)
|
|
154
|
+
return unless earliest_start && job.performed_at < earliest_start
|
|
155
|
+
|
|
156
|
+
duration = job.finished_at && (job.finished_at - job.performed_at)
|
|
157
|
+
attributes = { performed_at: earliest_start }
|
|
158
|
+
attributes[:finished_at] = earliest_start + duration if duration
|
|
159
|
+
if job.finished_at
|
|
160
|
+
attributes[:scheduled_at] = earliest_start
|
|
161
|
+
attributes[:updated_at] = attributes[:finished_at]
|
|
162
|
+
payload = job.serialized_params.deep_dup
|
|
163
|
+
payload["enqueued_at"] = earliest_start.utc.iso8601(9)
|
|
164
|
+
payload["scheduled_at"] = earliest_start.utc.iso8601(9)
|
|
165
|
+
attributes[:serialized_params] = payload
|
|
166
|
+
end
|
|
167
|
+
job.update_columns(attributes)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def recent_seed_time(day_offset, slot)
|
|
171
|
+
day_start = NOW.beginning_of_day - day_offset.days
|
|
172
|
+
return day_start + (30 + (slot * 73)).minutes unless day_offset.zero?
|
|
173
|
+
|
|
174
|
+
# Always stay inside today's calendar bucket without creating future rows,
|
|
175
|
+
# including when seeds run just after midnight.
|
|
176
|
+
elapsed = [(NOW - day_start).to_i - 1, 0].max
|
|
177
|
+
offset = elapsed.zero? ? 0 : ((slot + 1) * elapsed / 14.0).floor
|
|
178
|
+
day_start + [offset, elapsed].min.seconds
|
|
43
179
|
end
|
|
44
180
|
|
|
45
181
|
# 1. VideoProcessingPipeline (succeeded) — fan-out + fan-in DAG
|
|
@@ -70,7 +206,7 @@ report = create_pipeline(type: "ReportGenerationPipeline", status: "running",
|
|
|
70
206
|
params: { report_type: "monthly_revenue", month: "2026-02" }, age: 15.minutes)
|
|
71
207
|
report_steps = add_steps(report,
|
|
72
208
|
{ key: "query_data", job: "QueryDataJob" },
|
|
73
|
-
{ key: "aggregate", job: "AggregateJob", status: "enqueued" },
|
|
209
|
+
{ key: "aggregate", job: "AggregateJob", status: "enqueued", job_state: :running },
|
|
74
210
|
{ key: "render_pdf", job: "RenderPdfJob", status: "pending" },
|
|
75
211
|
{ key: "email_report", job: "EmailReportJob", status: "pending" })
|
|
76
212
|
add_edges(report, report_steps, %w[query_data aggregate], %w[aggregate render_pdf], %w[render_pdf email_report])
|
|
@@ -148,7 +284,7 @@ ingest2 = create_pipeline(type: "DataIngestionPipeline", status: "running",
|
|
|
148
284
|
strategy: "continue", age: 3.minutes)
|
|
149
285
|
ingest2_steps = add_steps(ingest2,
|
|
150
286
|
{ key: "extract", job: "ExtractJob" },
|
|
151
|
-
{ key: "validate", job: "ValidateJob", status: "enqueued" },
|
|
287
|
+
{ key: "validate", job: "ValidateJob", status: "enqueued", job_state: :queued },
|
|
152
288
|
{ key: "transform", job: "TransformJob", status: "pending" },
|
|
153
289
|
{ key: "load", job: "LoadJob", status: "pending" })
|
|
154
290
|
add_edges(ingest2, ingest2_steps, %w[extract validate], %w[validate transform], %w[transform load])
|
|
@@ -374,4 +510,182 @@ add_edges(notif_routing, notif_routing_steps,
|
|
|
374
510
|
%w[load_preferences notification_channel],
|
|
375
511
|
%w[notification_channel send_email], %w[notification_channel send_push])
|
|
376
512
|
|
|
377
|
-
|
|
513
|
+
# Fixture A: 139 steps and 224 edges. This independently exercises the
|
|
514
|
+
# dashboard's large-pipeline threshold without exceeding Mermaid's edge cap.
|
|
515
|
+
warehouse = create_pipeline(
|
|
516
|
+
type: "DataWarehouseRebuildPipeline",
|
|
517
|
+
status: "running",
|
|
518
|
+
params: { warehouse: "analytics", shards: 48 },
|
|
519
|
+
strategy: "continue",
|
|
520
|
+
age: 2.hours
|
|
521
|
+
)
|
|
522
|
+
warehouse_defs = [
|
|
523
|
+
{ key: "plan", job: "WarehousePlanJob", start_after: 10, duration: 20 }
|
|
524
|
+
]
|
|
525
|
+
|
|
526
|
+
48.times do |index|
|
|
527
|
+
suffix = index.to_s.rjust(2, "0")
|
|
528
|
+
warehouse_defs << {
|
|
529
|
+
key: "extract_#{suffix}", job: "WarehouseExtractJob",
|
|
530
|
+
start_after: 35 + ((index % 8) * 2), duration: 20 + (index % 5)
|
|
531
|
+
}
|
|
532
|
+
end
|
|
533
|
+
|
|
534
|
+
48.times do |index|
|
|
535
|
+
suffix = index.to_s.rjust(2, "0")
|
|
536
|
+
attributes = {
|
|
537
|
+
key: "transform_#{suffix}", job: "WarehouseTransformJob",
|
|
538
|
+
start_after: 90 + ((index % 8) * 3), duration: 30 + (index % 7)
|
|
539
|
+
}
|
|
540
|
+
if index < 36
|
|
541
|
+
warehouse_defs << attributes
|
|
542
|
+
elsif index < 39
|
|
543
|
+
warehouse_defs << attributes.merge(status: "enqueued", job_state: :running)
|
|
544
|
+
elsif index < 42
|
|
545
|
+
warehouse_defs << attributes.merge(status: "enqueued", job_state: :queued)
|
|
546
|
+
else
|
|
547
|
+
warehouse_defs << attributes.merge(status: "pending", job_state: :none)
|
|
548
|
+
end
|
|
549
|
+
end
|
|
550
|
+
|
|
551
|
+
warehouse_defs << {
|
|
552
|
+
key: "merge", job: "WarehouseMergeJob", status: "pending", job_state: :none,
|
|
553
|
+
start_after: 160, duration: 15
|
|
554
|
+
}
|
|
555
|
+
40.times do |index|
|
|
556
|
+
warehouse_defs << {
|
|
557
|
+
key: "load_#{index.to_s.rjust(2, '0')}", job: "WarehouseLoadJob",
|
|
558
|
+
status: "pending", job_state: :none, start_after: 180 + index, duration: 10
|
|
559
|
+
}
|
|
560
|
+
end
|
|
561
|
+
warehouse_defs << {
|
|
562
|
+
key: "finalize", job: "WarehouseFinalizeJob", status: "pending", job_state: :none,
|
|
563
|
+
start_after: 230, duration: 10
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
warehouse_steps = add_steps(warehouse, *warehouse_defs)
|
|
567
|
+
warehouse_edges = []
|
|
568
|
+
48.times do |index|
|
|
569
|
+
suffix = index.to_s.rjust(2, "0")
|
|
570
|
+
warehouse_edges << ["plan", "extract_#{suffix}"]
|
|
571
|
+
warehouse_edges << ["extract_#{suffix}", "transform_#{suffix}"]
|
|
572
|
+
warehouse_edges << ["transform_#{suffix}", "merge"]
|
|
573
|
+
end
|
|
574
|
+
40.times do |index|
|
|
575
|
+
suffix = index.to_s.rjust(2, "0")
|
|
576
|
+
warehouse_edges << ["merge", "load_#{suffix}"]
|
|
577
|
+
warehouse_edges << ["load_#{suffix}", "finalize"]
|
|
578
|
+
end
|
|
579
|
+
add_edges(warehouse, warehouse_steps, *warehouse_edges)
|
|
580
|
+
|
|
581
|
+
# Fixture B: 46 nodes connected to every subsequent node, producing exactly
|
|
582
|
+
# C(46, 2) = 1,035 edges while remaining below the 60-step large-pipeline gate.
|
|
583
|
+
dense = create_pipeline(
|
|
584
|
+
type: "DenseFanoutPipeline",
|
|
585
|
+
status: "succeeded",
|
|
586
|
+
params: { nodes: 46, topology: "fully_connected_forward" },
|
|
587
|
+
strategy: "ignore",
|
|
588
|
+
age: 26.hours,
|
|
589
|
+
duration: 12.minutes
|
|
590
|
+
)
|
|
591
|
+
dense_defs = 46.times.map do |index|
|
|
592
|
+
{
|
|
593
|
+
key: "node_#{index.to_s.rjust(2, '0')}", job: "DenseFanoutNodeJob",
|
|
594
|
+
start_after: 5 + (index * 10), duration: 5
|
|
595
|
+
}
|
|
596
|
+
end
|
|
597
|
+
dense_steps = add_steps(dense, *dense_defs)
|
|
598
|
+
dense_edges = 46.times.flat_map do |from|
|
|
599
|
+
((from + 1)...46).map do |to|
|
|
600
|
+
["node_#{from.to_s.rjust(2, '0')}", "node_#{to.to_s.rjust(2, '0')}"]
|
|
601
|
+
end
|
|
602
|
+
end
|
|
603
|
+
add_edges(dense, dense_steps, *dense_edges)
|
|
604
|
+
|
|
605
|
+
# A non-terminal execution is intentionally older than GoodJob's normal
|
|
606
|
+
# retention window. Its dangling IDs model job rows that cleanup has removed.
|
|
607
|
+
retention_gap = create_pipeline(
|
|
608
|
+
type: "LongRunningArchivePipeline",
|
|
609
|
+
status: "running",
|
|
610
|
+
params: { archive: "legal-hold", partitions: 4 },
|
|
611
|
+
strategy: "continue",
|
|
612
|
+
age: 20.days
|
|
613
|
+
)
|
|
614
|
+
retention_steps = add_steps(
|
|
615
|
+
retention_gap,
|
|
616
|
+
{ key: "scan", job: "ArchiveScanJob", job_state: :dangling },
|
|
617
|
+
{ key: "compact", job: "ArchiveCompactJob", job_state: :dangling },
|
|
618
|
+
{ key: "verify", job: "ArchiveVerifyJob", status: "enqueued", job_state: :dangling },
|
|
619
|
+
{ key: "publish", job: "ArchivePublishJob", status: "pending", job_state: :dangling }
|
|
620
|
+
)
|
|
621
|
+
add_edges(retention_gap, retention_steps, %w[scan compact], %w[compact verify], %w[verify publish])
|
|
622
|
+
|
|
623
|
+
# Add 180 compact historical executions. Together with the original 20 and the
|
|
624
|
+
# three scale/retention fixtures above, the dashboard has 203 rows to paginate.
|
|
625
|
+
# The generated set is exactly 80/8/5/4/3 percent (rounded) across statuses.
|
|
626
|
+
history_statuses = (["succeeded"] * 144) + (["running"] * 14) + (["failed"] * 9) +
|
|
627
|
+
(["halted"] * 7) + (["skipped"] * 6)
|
|
628
|
+
history_types = %w[
|
|
629
|
+
CatalogRefreshPipeline CustomerImportPipeline DailyMetricsPipeline
|
|
630
|
+
InventorySnapshotPipeline SearchReindexPipeline WebhookReplayPipeline
|
|
631
|
+
]
|
|
632
|
+
|
|
633
|
+
180.times do |index|
|
|
634
|
+
status = history_statuses[(index * 73) % history_statuses.length]
|
|
635
|
+
day_offset = index % 14
|
|
636
|
+
slot = index / 14
|
|
637
|
+
duration = status == "running" ? nil : (45 + ((index * 17) % 480)).seconds
|
|
638
|
+
pipeline = create_pipeline(
|
|
639
|
+
type: history_types[index % history_types.length],
|
|
640
|
+
status: status,
|
|
641
|
+
params: { run: index + 1, shard: index % 12, source: "demo" },
|
|
642
|
+
strategy: %w[halt continue ignore][index % 3],
|
|
643
|
+
created_at: recent_seed_time(day_offset, slot),
|
|
644
|
+
duration: duration,
|
|
645
|
+
halt_triggered: status == "halted"
|
|
646
|
+
)
|
|
647
|
+
|
|
648
|
+
step_defs = case status
|
|
649
|
+
when "succeeded"
|
|
650
|
+
[
|
|
651
|
+
{ key: "prepare", job: "DemoPrepareJob", start_after: 5, duration: 8 },
|
|
652
|
+
{ key: "process", job: "DemoProcessJob", start_after: 18, duration: 12 },
|
|
653
|
+
{ key: "commit", job: "DemoCommitJob", start_after: 35, duration: 7 }
|
|
654
|
+
]
|
|
655
|
+
when "running"
|
|
656
|
+
active_state = index.even? ? :running : :queued
|
|
657
|
+
[
|
|
658
|
+
{ key: "prepare", job: "DemoPrepareJob", start_after: 5, duration: 8 },
|
|
659
|
+
{ key: "process", job: "DemoProcessJob", status: "enqueued",
|
|
660
|
+
job_state: active_state, start_after: 18, duration: 12 },
|
|
661
|
+
{ key: "commit", job: "DemoCommitJob", status: "pending", job_state: :none }
|
|
662
|
+
]
|
|
663
|
+
when "failed"
|
|
664
|
+
[
|
|
665
|
+
{ key: "prepare", job: "DemoPrepareJob", start_after: 5, duration: 8 },
|
|
666
|
+
{ key: "process", job: "DemoProcessJob", status: "failed", start_after: 18, duration: 12,
|
|
667
|
+
error_class: "DemoImportError", error_message: "Synthetic row #{index + 1} could not be imported" },
|
|
668
|
+
{ key: "commit", job: "DemoCommitJob", status: "skipped", job_state: :none }
|
|
669
|
+
]
|
|
670
|
+
when "halted"
|
|
671
|
+
[
|
|
672
|
+
{ key: "prepare", job: "DemoPrepareJob", status: "failed", start_after: 5, duration: 8,
|
|
673
|
+
error_class: "DemoHalt", error_message: "Synthetic safety check halted the execution" },
|
|
674
|
+
{ key: "process", job: "DemoProcessJob", status: "halted", start_after: 18, duration: 4 },
|
|
675
|
+
{ key: "commit", job: "DemoCommitJob", status: "skipped", job_state: :none }
|
|
676
|
+
]
|
|
677
|
+
else
|
|
678
|
+
[
|
|
679
|
+
{ key: "prepare", job: "DemoPrepareJob", status: "skipped", job_state: :none },
|
|
680
|
+
{ key: "process", job: "DemoProcessJob", status: "skipped", job_state: :none },
|
|
681
|
+
{ key: "commit", job: "DemoCommitJob", status: "skipped", job_state: :none }
|
|
682
|
+
]
|
|
683
|
+
end
|
|
684
|
+
|
|
685
|
+
steps = add_steps(pipeline, *step_defs)
|
|
686
|
+
add_edges(pipeline, steps, %w[prepare process], %w[process commit])
|
|
687
|
+
end
|
|
688
|
+
|
|
689
|
+
puts "Done! #{GoodPipeline::PipelineRecord.count} pipelines, " \
|
|
690
|
+
"#{GoodPipeline::StepRecord.count} steps, and " \
|
|
691
|
+
"#{GoodJob::Job.where(queue_name: SEED_JOB_QUEUE).count} synthetic jobs seeded."
|
|
Binary file
|
|
Binary file
|