good_pipeline 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. checksums.yaml +4 -4
  2. data/Appraisals +25 -0
  3. data/CHANGELOG.md +58 -0
  4. data/README.md +67 -11
  5. data/app/controllers/good_pipeline/frontends_controller.rb +18 -1
  6. data/app/controllers/good_pipeline/pipelines_controller.rb +254 -36
  7. data/app/controllers/good_pipeline/themes_controller.rb +15 -0
  8. data/app/frontend/good_pipeline/dashboard.css +407 -0
  9. data/app/frontend/good_pipeline/dashboard.js +542 -0
  10. data/app/helpers/good_pipeline/mermaid_diagram_builder.rb +225 -66
  11. data/app/helpers/good_pipeline/pipelines_helper.rb +89 -5
  12. data/app/jobs/good_pipeline/pipeline_callback_job.rb +2 -2
  13. data/app/jobs/good_pipeline/pipeline_reconciliation_job.rb +1 -2
  14. data/app/jobs/good_pipeline/step_finished_job.rb +6 -2
  15. data/app/models/good_pipeline/pipeline_record.rb +7 -4
  16. data/app/models/good_pipeline/step_record.rb +6 -3
  17. data/app/views/good_pipeline/pipelines/_actions.html.erb +28 -0
  18. data/app/views/good_pipeline/pipelines/_chain_links.html.erb +17 -29
  19. data/app/views/good_pipeline/pipelines/_diagram.html.erb +101 -0
  20. data/app/views/good_pipeline/pipelines/_filter_bar.html.erb +51 -0
  21. data/app/views/good_pipeline/pipelines/_kpi_strip.html.erb +74 -0
  22. data/app/views/good_pipeline/pipelines/_pager.html.erb +31 -0
  23. data/app/views/good_pipeline/pipelines/_row.html.erb +103 -0
  24. data/app/views/good_pipeline/pipelines/_row_detail.html.erb +93 -0
  25. data/app/views/good_pipeline/pipelines/_sidebar.html.erb +115 -0
  26. data/app/views/good_pipeline/pipelines/_steps_table.html.erb +46 -33
  27. data/app/views/good_pipeline/pipelines/_table.html.erb +33 -0
  28. data/app/views/good_pipeline/pipelines/_timeline.html.erb +55 -0
  29. data/app/views/good_pipeline/pipelines/_topbar.html.erb +54 -0
  30. data/app/views/good_pipeline/pipelines/definitions.html.erb +88 -55
  31. data/app/views/good_pipeline/pipelines/index.html.erb +31 -42
  32. data/app/views/good_pipeline/pipelines/show.html.erb +54 -47
  33. data/app/views/layouts/good_pipeline/application.html.erb +24 -153
  34. data/config/routes.rb +4 -0
  35. data/demo/config/environments/development.rb +3 -0
  36. data/demo/config/environments/test.rb +2 -1
  37. data/demo/db/migrate/20260319205325_create_good_jobs.rb +1 -1
  38. data/demo/db/migrate/20260319205326_create_good_pipeline_tables.rb +1 -1
  39. data/demo/db/migrate/20260810000000_add_good_pipeline_dashboard_indexes.rb +20 -0
  40. data/demo/db/seeds.rb +325 -11
  41. data/demo/docs/screenshots/definitions.png +0 -0
  42. data/demo/docs/screenshots/show.png +0 -0
  43. data/demo/test/good_pipeline/test_bulk_enqueue.rb +54 -1
  44. data/demo/test/good_pipeline/test_cancellation.rb +312 -0
  45. data/demo/test/good_pipeline/test_cleanup.rb +31 -1
  46. data/demo/test/good_pipeline/test_coordinator.rb +39 -0
  47. data/demo/test/good_pipeline/test_dashboard.rb +667 -0
  48. data/demo/test/good_pipeline/test_dashboard_rerun.rb +244 -0
  49. data/demo/test/good_pipeline/test_pipeline_callback_job.rb +11 -0
  50. data/demo/test/good_pipeline/test_pipeline_record.rb +30 -0
  51. data/demo/test/good_pipeline/test_runner.rb +40 -1
  52. data/demo/test/good_pipeline/test_step_record.rb +37 -0
  53. data/demo/test/integration/test_barrier_execution.rb +502 -0
  54. data/demo/test/integration/test_chain_locking.rb +54 -0
  55. data/demo/test/integration/test_pipeline_chaining.rb +96 -0
  56. data/demo/test/test_helper.rb +20 -0
  57. data/docs/architecture.md +45 -29
  58. data/docs/branching.md +4 -0
  59. data/docs/callbacks.md +9 -7
  60. data/docs/cleanup.md +2 -2
  61. data/docs/dag-validation.md +16 -0
  62. data/docs/dashboard.md +84 -22
  63. data/docs/defining-pipelines.md +34 -1
  64. data/docs/failure-strategies.md +24 -1
  65. data/docs/getting-started.md +9 -1
  66. data/docs/index.md +1 -1
  67. data/docs/introduction.md +4 -4
  68. data/docs/monitoring.md +8 -3
  69. data/docs/pipeline-chaining.md +17 -9
  70. data/docs/public/screenshots/definitions.png +0 -0
  71. data/docs/public/screenshots/index.png +0 -0
  72. data/docs/public/screenshots/show.png +0 -0
  73. data/docs/screenshots/definitions.png +0 -0
  74. data/docs/screenshots/index.png +0 -0
  75. data/docs/screenshots/show.png +0 -0
  76. data/gemfiles/rails_7_2.gemfile +24 -0
  77. data/gemfiles/rails_7_2.gemfile.lock +412 -0
  78. data/gemfiles/rails_8_0.gemfile +24 -0
  79. data/gemfiles/rails_8_0.gemfile.lock +411 -0
  80. data/gemfiles/rails_8_1.gemfile +24 -0
  81. data/gemfiles/rails_8_1.gemfile.lock +415 -0
  82. data/lib/generators/good_pipeline/install/templates/create_good_pipeline_tables.rb.erb +9 -0
  83. data/lib/generators/good_pipeline/upgrade/templates/add_good_pipeline_dashboard_indexes.rb.erb +27 -0
  84. data/lib/generators/good_pipeline/upgrade/upgrade_generator.rb +32 -0
  85. data/lib/good_pipeline/barrier_compiler.rb +106 -0
  86. data/lib/good_pipeline/chain_coordinator.rb +9 -5
  87. data/lib/good_pipeline/constants.rb +2 -0
  88. data/lib/good_pipeline/coordinator.rb +269 -140
  89. data/lib/good_pipeline/dashboard/connection_info.rb +67 -0
  90. data/lib/good_pipeline/dashboard/definition_stages.rb +59 -0
  91. data/lib/good_pipeline/dashboard/filter_set.rb +77 -0
  92. data/lib/good_pipeline/dashboard/kpi_calculator.rb +143 -0
  93. data/lib/good_pipeline/dashboard/sparkline.rb +73 -0
  94. data/lib/good_pipeline/dashboard/stage_lanes.rb +307 -0
  95. data/lib/good_pipeline/dashboard/step_timings.rb +50 -0
  96. data/lib/good_pipeline/dashboard.rb +21 -0
  97. data/lib/good_pipeline/errors.rb +11 -0
  98. data/lib/good_pipeline/pipeline.rb +14 -2
  99. data/lib/good_pipeline/step_definition.rb +16 -1
  100. data/lib/good_pipeline/version.rb +1 -1
  101. data/lib/good_pipeline.rb +7 -1
  102. data/mise.toml +1 -1
  103. metadata +48 -13
  104. data/app/frontend/good_pipeline/style.css +0 -597
  105. data/app/views/good_pipeline/pipelines/_pagination.html.erb +0 -24
  106. data/app/views/good_pipeline/pipelines/_pipeline_row.html.erb +0 -7
@@ -1,155 +1,26 @@
1
- <!DOCTYPE html>
2
- <html lang="en" data-theme="light">
3
- <head>
4
- <meta charset="utf-8">
5
- <meta name="viewport" content="width=device-width, initial-scale=1">
6
- <title>GoodPipeline</title>
7
- <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@picocss/pico@2/css/pico.min.css">
8
- <link rel="stylesheet" href="<%= frontend_static_path(id: 'style', format: :css) %>">
9
- </head>
10
- <body>
11
- <header class="dashboard-header">
12
- <span class="header-left">
13
- <%= link_to "GoodPipeline", root_path %>
14
- <%= link_to "Definitions", definitions_pipelines_path, class: "header-nav-link" %>
15
- </span>
16
- <span class="header-right">
17
- <span class="header-version">v<%= GoodPipeline::VERSION %></span>
18
- <a href="https://github.com/milkstrawai/good_pipeline" target="_blank" rel="noopener" class="header-github" title="GitHub">
19
- <svg height="18" width="18" viewBox="0 0 16 16" fill="currentColor"><path d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"/></svg>
20
- </a>
21
- </span>
22
- </header>
23
- <main class="dashboard-content">
1
+ <% theme = %w[dark light].include?(cookies[:good_pipeline_theme]) ? cookies[:good_pipeline_theme] : "dark" %>
2
+ <!doctype html>
3
+ <html lang="en" data-gp-theme="<%= theme %>">
4
+ <head>
5
+ <meta charset="utf-8">
6
+ <meta name="viewport" content="width=device-width, initial-scale=1">
7
+ <meta name="csrf-token" content="<%= form_authenticity_token %>">
8
+ <meta name="good-pipeline-theme-url" content="<%= theme_path %>">
9
+ <meta name="turbo-visit-control" content="reload">
10
+ <title>GoodPipeline</title>
11
+ <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'%3E%3Crect width='32' height='32' rx='6' fill='%23111317'/%3E%3Ctext x='5' y='22' fill='%23e6e9ef' font-family='monospace' font-size='16'%3Egp%3C/text%3E%3C/svg%3E">
12
+
13
+ <link rel="preconnect" href="https://fonts.googleapis.com">
14
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
15
+ <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&amp;family=JetBrains+Mono:wght@400;500&amp;display=swap" rel="stylesheet">
16
+ <link rel="stylesheet" href="<%= frontend_static_path(id: "dashboard", format: :css) %>">
17
+ <script src="https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.min.js" defer nonce="<%= content_security_policy_nonce %>"></script>
18
+ <script src="<%= frontend_static_path(id: "dashboard", format: :js) %>" defer nonce="<%= content_security_policy_nonce %>"></script>
19
+ </head>
20
+ <body>
21
+ <% if flash[:notice] %><div class="gp-flash gp-flash--notice" role="status"><%= flash[:notice] %></div><% end %>
22
+ <% if flash[:alert] %><div class="gp-flash gp-flash--alert" role="alert"><%= flash[:alert] %></div><% end %>
24
23
  <%= yield %>
25
- </main>
26
- <script src="https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.min.js"></script>
27
- <script>
28
- mermaid.initialize({
29
- startOnLoad: true,
30
- theme: 'neutral',
31
- flowchart: { curve: 'basis', nodeSpacing: 30, rankSpacing: 50 }
32
- });
33
- document.querySelectorAll('tr[data-href]').forEach(function(row) {
34
- row.addEventListener('click', function() {
35
- window.location = this.dataset.href;
36
- });
37
- });
38
- </script>
39
- <script>
40
- (function() {
41
- var MIN_SCALE = 0.5, MAX_SCALE = 4, ZOOM_STEP = 0.25;
42
- var states = new WeakMap();
43
-
44
- function getState(container) {
45
- if (!states.has(container)) states.set(container, { scale: 1, panX: 0, panY: 0 });
46
- return states.get(container);
47
- }
48
-
49
- function applyTransform(container) {
50
- var s = getState(container);
51
- var canvas = container.querySelector('.diagram-canvas');
52
- if (canvas) canvas.style.transform = 'translate(' + s.panX + 'px, ' + s.panY + 'px) scale(' + s.scale + ')';
53
- }
54
-
55
- function clampScale(val) {
56
- return Math.min(MAX_SCALE, Math.max(MIN_SCALE, val));
57
- }
58
-
59
- function zoomAroundPoint(s, centerX, centerY, newScale) {
60
- var oldScale = s.scale;
61
- newScale = clampScale(newScale);
62
- if (newScale === oldScale) return;
63
- var ratio = newScale / oldScale;
64
- s.panX = centerX - (centerX - s.panX) * ratio;
65
- s.panY = centerY - (centerY - s.panY) * ratio;
66
- s.scale = newScale;
67
- }
68
-
69
- // Button actions via event delegation
70
- document.addEventListener('click', function(e) {
71
- var btn = e.target.closest('[data-action]');
72
- if (!btn) return;
73
- var container = btn.closest('.diagram-container');
74
- if (!container) return;
75
- var s = getState(container);
76
- var action = btn.dataset.action;
77
-
78
- if (action === 'zoom-in' || action === 'zoom-out') {
79
- var viewport = container.querySelector('.diagram-viewport');
80
- var rect = viewport.getBoundingClientRect();
81
- var centerX = rect.width / 2;
82
- var centerY = rect.height / 2;
83
- var delta = action === 'zoom-in' ? ZOOM_STEP : -ZOOM_STEP;
84
- zoomAroundPoint(s, centerX, centerY, s.scale + delta);
85
- } else if (action === 'zoom-reset') {
86
- s.scale = 1; s.panX = 0; s.panY = 0;
87
- } else if (action === 'fullscreen') {
88
- if (document.fullscreenElement === container) {
89
- document.exitFullscreen();
90
- } else {
91
- container.requestFullscreen();
92
- }
93
- return;
94
- }
95
- applyTransform(container);
96
- });
97
-
98
- // Mouse wheel zoom toward cursor
99
- document.querySelectorAll('.diagram-viewport').forEach(function(viewport) {
100
- viewport.addEventListener('wheel', function(e) {
101
- e.preventDefault();
102
- var container = viewport.closest('.diagram-container');
103
- var s = getState(container);
104
- var rect = viewport.getBoundingClientRect();
105
- var cursorX = e.clientX - rect.left;
106
- var cursorY = e.clientY - rect.top;
107
- var delta = e.deltaY < 0 ? ZOOM_STEP : -ZOOM_STEP;
108
- zoomAroundPoint(s, cursorX, cursorY, s.scale + delta);
109
- applyTransform(container);
110
- }, { passive: false });
111
- });
112
-
113
- // Click-drag pan
114
- document.querySelectorAll('.diagram-viewport').forEach(function(viewport) {
115
- viewport.addEventListener('mousedown', function(e) {
116
- if (e.button !== 0) return;
117
- var container = viewport.closest('.diagram-container');
118
- var canvas = container.querySelector('.diagram-canvas');
119
- var s = getState(container);
120
- var startX = e.clientX - s.panX;
121
- var startY = e.clientY - s.panY;
122
- canvas.classList.add('dragging');
123
-
124
- function onMove(ev) {
125
- s.panX = ev.clientX - startX;
126
- s.panY = ev.clientY - startY;
127
- applyTransform(container);
128
- }
129
- function onUp() {
130
- canvas.classList.remove('dragging');
131
- document.removeEventListener('mousemove', onMove);
132
- document.removeEventListener('mouseup', onUp);
133
- }
134
- document.addEventListener('mousemove', onMove);
135
- document.addEventListener('mouseup', onUp);
136
- });
137
- });
138
-
139
- // Reset zoom when switching definitions
140
- document.querySelectorAll('.definition-item').forEach(function(btn) {
141
- btn.addEventListener('click', function() {
142
- var type = btn.dataset.pipeline;
143
- var panel = document.querySelector('.definition-panel[data-pipeline="' + type + '"]');
144
- if (!panel) return;
145
- var container = panel.querySelector('.diagram-container');
146
- if (!container) return;
147
- var s = getState(container);
148
- s.scale = 1; s.panX = 0; s.panY = 0;
149
- applyTransform(container);
150
- });
151
- });
152
- })();
153
- </script>
154
- </body>
24
+ <div id="gp-aria-live" class="gp-sr-only" aria-live="polite" aria-atomic="true"></div>
25
+ </body>
155
26
  </html>
data/config/routes.rb CHANGED
@@ -5,8 +5,12 @@ GoodPipeline::Engine.routes.draw do
5
5
 
6
6
  resources :pipelines, only: %i[index show] do
7
7
  get :definitions, on: :collection
8
+ post :cancel, on: :member
9
+ post :rerun, on: :member
8
10
  end
9
11
 
12
+ patch :theme, to: "themes#update", as: :theme
13
+
10
14
  scope :frontend, controller: :frontends, defaults: { version: GoodPipeline::VERSION.tr(".", "-") } do
11
15
  get "static/:version/:id", action: :static, as: :frontend_static
12
16
  end
@@ -7,3 +7,6 @@ Rails.application.configure do
7
7
  config.active_support.deprecation = :log
8
8
  config.good_job.execution_mode = :async
9
9
  end
10
+
11
+ # The demo is a local development application; expose its pipeline controls.
12
+ GoodPipeline.dashboard_mutations_enabled = true
@@ -4,7 +4,8 @@ Rails.application.configure do
4
4
  config.cache_classes = true
5
5
  config.eager_load = false
6
6
  config.consider_all_requests_local = true
7
- config.action_controller.perform_caching = false if config.respond_to?(:action_controller)
7
+ config.action_controller.perform_caching = true if config.respond_to?(:action_controller)
8
+ config.cache_store = :memory_store
8
9
  config.active_support.deprecation = :stderr
9
10
  config.good_job.execution_mode = :external
10
11
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- class CreateGoodJobs < ActiveRecord::Migration[8.1]
3
+ class CreateGoodJobs < ActiveRecord::Migration[7.2]
4
4
  def change
5
5
  # Uncomment for Postgres v12 or earlier to enable gen_random_uuid() support
6
6
  # enable_extension 'pgcrypto'
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- class CreateGoodPipelineTables < ActiveRecord::Migration[8.1]
3
+ class CreateGoodPipelineTables < ActiveRecord::Migration[7.2]
4
4
  def change
5
5
  # Uncomment for Postgres v12 or earlier to enable gen_random_uuid() support
6
6
  # enable_extension 'pgcrypto'
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ class AddGoodPipelineDashboardIndexes < ActiveRecord::Migration[7.2]
4
+ disable_ddl_transaction!
5
+
6
+ def change
7
+ add_index :good_pipeline_pipelines, %i[type created_at id],
8
+ order: { created_at: :desc, id: :desc },
9
+ name: :index_gp_pipelines_on_type_created_at_id,
10
+ algorithm: :concurrently, if_not_exists: true
11
+ add_index :good_pipeline_pipelines, %i[status created_at id],
12
+ order: { created_at: :desc, id: :desc },
13
+ name: :index_gp_pipelines_on_status_created_at_id,
14
+ algorithm: :concurrently, if_not_exists: true
15
+ add_index :good_pipeline_pipelines, %i[created_at id],
16
+ order: { created_at: :desc, id: :desc },
17
+ name: :index_gp_pipelines_on_created_at_id,
18
+ algorithm: :concurrently, if_not_exists: true
19
+ end
20
+ end
data/demo/db/seeds.rb CHANGED
@@ -1,36 +1,115 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "securerandom"
4
+
3
5
  puts "Seeding GoodPipeline demo data..."
4
6
 
7
+ SEED_JOB_QUEUE = "good_pipeline_demo_seed"
8
+ SEED_FUTURE = Time.utc(2100, 1, 1)
9
+ NOW = Time.current.change(usec: 0)
10
+
11
+ # GoodJob rows have no foreign key from GoodPipeline, so tag synthetic rows by
12
+ # queue name and remove them explicitly before rebuilding the demo fixtures.
13
+ GoodJob::Job.where(queue_name: SEED_JOB_QUEUE).delete_all
5
14
  GoodPipeline::ChainRecord.delete_all
6
15
  GoodPipeline::DependencyRecord.delete_all
7
16
  GoodPipeline::StepRecord.delete_all
8
17
  GoodPipeline::PipelineRecord.delete_all
9
18
 
10
- NOW = Time.current # rubocop:disable Lint/ConstantDefinitionInBlock
11
-
12
- def create_pipeline(type:, status:, params:, strategy: "halt", age:, duration: nil, halt_triggered: false) # rubocop:disable Metrics/ParameterLists
19
+ def create_pipeline(type:, status:, params:, strategy: "halt", age: nil, created_at: nil, duration: nil,
20
+ halt_triggered: false)
21
+ timestamp = created_at || (NOW - age)
13
22
  pipeline = GoodPipeline::PipelineRecord.create!(
14
23
  type: type, status: status, params: params,
15
24
  on_failure_strategy: strategy, halt_triggered: halt_triggered
16
25
  )
17
26
  pipeline.update_columns(
18
- created_at: NOW - age,
19
- updated_at: duration ? NOW - age + duration : NOW - age
27
+ created_at: timestamp,
28
+ updated_at: duration ? timestamp + duration : timestamp
20
29
  )
21
30
  pipeline
22
31
  end
23
32
 
33
+ def active_job_payload(job_id:, job_class:, scheduled_at:, enqueued_at:)
34
+ {
35
+ "job_class" => job_class,
36
+ "job_id" => job_id,
37
+ "provider_job_id" => job_id,
38
+ "queue_name" => SEED_JOB_QUEUE,
39
+ "priority" => 0,
40
+ "arguments" => [],
41
+ "executions" => 0,
42
+ "exception_executions" => {},
43
+ "locale" => "en",
44
+ "timezone" => "UTC",
45
+ "enqueued_at" => enqueued_at.utc.iso8601(9),
46
+ "scheduled_at" => scheduled_at.utc.iso8601(9)
47
+ }
48
+ end
49
+
50
+ def create_seed_job(job_class:, state:, created_at:, duration:, error: nil)
51
+ job_id = SecureRandom.uuid
52
+ unfinished = %i[running queued].include?(state)
53
+ scheduled_at = unfinished ? SEED_FUTURE : created_at
54
+ performed_at = %i[completed running].include?(state) ? created_at : nil
55
+ finished_at = state == :completed ? created_at + duration : nil
56
+
57
+ GoodJob::Job.create!(
58
+ id: job_id,
59
+ active_job_id: job_id,
60
+ queue_name: SEED_JOB_QUEUE,
61
+ priority: 0,
62
+ serialized_params: active_job_payload(
63
+ job_id: job_id,
64
+ job_class: job_class,
65
+ scheduled_at: scheduled_at,
66
+ enqueued_at: created_at
67
+ ),
68
+ scheduled_at: scheduled_at,
69
+ performed_at: performed_at,
70
+ finished_at: finished_at,
71
+ error: error,
72
+ executions_count: state == :queued ? 0 : 1,
73
+ job_class: job_class,
74
+ created_at: created_at,
75
+ updated_at: finished_at || created_at
76
+ )
77
+ end
78
+
79
+ def inferred_job_state(status)
80
+ case status
81
+ when "succeeded", "failed", "halted" then :completed
82
+ when "enqueued" then :queued
83
+ else :none
84
+ end
85
+ end
86
+
24
87
  def add_steps(pipeline, *step_defs)
25
88
  records = {}
26
- step_defs.each do |definition|
89
+ step_defs.each_with_index do |definition, index|
27
90
  branch_hash = definition.slice(:decides, :branch_result, :empty_arms, :branch_key, :branch_arm).transform_keys(&:to_s)
91
+ status = definition.fetch(:status, "succeeded")
92
+ job_state = definition.fetch(:job_state, inferred_job_state(status))
93
+ started_at = pipeline.created_at + definition.fetch(:start_after, (index + 1) * 5).seconds
94
+ job_duration = definition.fetch(:duration, 10).seconds
95
+ error = if status == "failed"
96
+ "#{definition.fetch(:error_class, 'RuntimeError')}: #{definition.fetch(:error_message, 'Synthetic failure')}"
97
+ end
98
+ good_job = if %i[completed running queued].include?(job_state)
99
+ create_seed_job(
100
+ job_class: definition[:job], state: job_state, created_at: started_at,
101
+ duration: job_duration, error: error
102
+ )
103
+ end
104
+ good_job_id = job_state == :dangling ? SecureRandom.uuid : good_job&.id
28
105
 
29
106
  records[definition[:key]] = GoodPipeline::StepRecord.create!(
30
107
  pipeline: pipeline, key: definition[:key], job_class: definition[:job],
31
- coordination_status: definition.fetch(:status, "succeeded"),
108
+ coordination_status: status, good_job_id: good_job_id,
32
109
  error_class: definition[:error_class], error_message: definition[:error_message],
33
- branch: branch_hash
110
+ branch: branch_hash,
111
+ created_at: pipeline.created_at + index.fdiv(1_000_000),
112
+ updated_at: pipeline.created_at + index.fdiv(1_000_000)
34
113
  )
35
114
  end
36
115
  records
@@ -40,6 +119,63 @@ def add_edges(pipeline, steps, *edges)
40
119
  edges.each do |from, to|
41
120
  GoodPipeline::DependencyRecord.create!(pipeline: pipeline, step: steps[to], depends_on_step: steps[from])
42
121
  end
122
+ normalize_job_timings(pipeline)
123
+ end
124
+
125
+ def normalize_job_timings(pipeline)
126
+ ordered_steps = pipeline.steps.order(:created_at, :id).to_a
127
+ dependencies = pipeline.dependencies.to_a
128
+ jobs = GoodJob::Job.where(id: ordered_steps.filter_map(&:good_job_id)).index_by(&:id)
129
+ parents = dependencies.group_by(&:step_id).transform_values do |records|
130
+ records.map(&:depends_on_step_id)
131
+ end
132
+ by_id = ordered_steps.index_by(&:id)
133
+ remaining = ordered_steps.to_h { |step| [step.id, parents.fetch(step.id, []).length] }
134
+ children = dependencies.group_by(&:depends_on_step_id)
135
+ ready = ordered_steps.select { |step| remaining.fetch(step.id).zero? }
136
+
137
+ until ready.empty?
138
+ step = ready.shift
139
+ job = jobs[step.good_job_id]
140
+ upstream_finishes = parents.fetch(step.id, []).filter_map do |id|
141
+ upstream = by_id[id]
142
+ jobs[upstream&.good_job_id]&.finished_at
143
+ end
144
+ shift_seed_job_after(job, upstream_finishes.max) if job&.performed_at && upstream_finishes.any?
145
+
146
+ children.fetch(step.id, []).each do |dependency|
147
+ remaining[dependency.step_id] -= 1
148
+ ready << by_id.fetch(dependency.step_id) if remaining.fetch(dependency.step_id).zero?
149
+ end
150
+ end
151
+ end
152
+
153
+ def shift_seed_job_after(job, earliest_start)
154
+ return unless earliest_start && job.performed_at < earliest_start
155
+
156
+ duration = job.finished_at && (job.finished_at - job.performed_at)
157
+ attributes = { performed_at: earliest_start }
158
+ attributes[:finished_at] = earliest_start + duration if duration
159
+ if job.finished_at
160
+ attributes[:scheduled_at] = earliest_start
161
+ attributes[:updated_at] = attributes[:finished_at]
162
+ payload = job.serialized_params.deep_dup
163
+ payload["enqueued_at"] = earliest_start.utc.iso8601(9)
164
+ payload["scheduled_at"] = earliest_start.utc.iso8601(9)
165
+ attributes[:serialized_params] = payload
166
+ end
167
+ job.update_columns(attributes)
168
+ end
169
+
170
+ def recent_seed_time(day_offset, slot)
171
+ day_start = NOW.beginning_of_day - day_offset.days
172
+ return day_start + (30 + (slot * 73)).minutes unless day_offset.zero?
173
+
174
+ # Always stay inside today's calendar bucket without creating future rows,
175
+ # including when seeds run just after midnight.
176
+ elapsed = [(NOW - day_start).to_i - 1, 0].max
177
+ offset = elapsed.zero? ? 0 : ((slot + 1) * elapsed / 14.0).floor
178
+ day_start + [offset, elapsed].min.seconds
43
179
  end
44
180
 
45
181
  # 1. VideoProcessingPipeline (succeeded) — fan-out + fan-in DAG
@@ -70,7 +206,7 @@ report = create_pipeline(type: "ReportGenerationPipeline", status: "running",
70
206
  params: { report_type: "monthly_revenue", month: "2026-02" }, age: 15.minutes)
71
207
  report_steps = add_steps(report,
72
208
  { key: "query_data", job: "QueryDataJob" },
73
- { key: "aggregate", job: "AggregateJob", status: "enqueued" },
209
+ { key: "aggregate", job: "AggregateJob", status: "enqueued", job_state: :running },
74
210
  { key: "render_pdf", job: "RenderPdfJob", status: "pending" },
75
211
  { key: "email_report", job: "EmailReportJob", status: "pending" })
76
212
  add_edges(report, report_steps, %w[query_data aggregate], %w[aggregate render_pdf], %w[render_pdf email_report])
@@ -148,7 +284,7 @@ ingest2 = create_pipeline(type: "DataIngestionPipeline", status: "running",
148
284
  strategy: "continue", age: 3.minutes)
149
285
  ingest2_steps = add_steps(ingest2,
150
286
  { key: "extract", job: "ExtractJob" },
151
- { key: "validate", job: "ValidateJob", status: "enqueued" },
287
+ { key: "validate", job: "ValidateJob", status: "enqueued", job_state: :queued },
152
288
  { key: "transform", job: "TransformJob", status: "pending" },
153
289
  { key: "load", job: "LoadJob", status: "pending" })
154
290
  add_edges(ingest2, ingest2_steps, %w[extract validate], %w[validate transform], %w[transform load])
@@ -374,4 +510,182 @@ add_edges(notif_routing, notif_routing_steps,
374
510
  %w[load_preferences notification_channel],
375
511
  %w[notification_channel send_email], %w[notification_channel send_push])
376
512
 
377
- puts "Done! #{GoodPipeline::PipelineRecord.count} pipelines seeded."
513
+ # Fixture A: 139 steps and 224 edges. This independently exercises the
514
+ # dashboard's large-pipeline threshold without exceeding Mermaid's edge cap.
515
+ warehouse = create_pipeline(
516
+ type: "DataWarehouseRebuildPipeline",
517
+ status: "running",
518
+ params: { warehouse: "analytics", shards: 48 },
519
+ strategy: "continue",
520
+ age: 2.hours
521
+ )
522
+ warehouse_defs = [
523
+ { key: "plan", job: "WarehousePlanJob", start_after: 10, duration: 20 }
524
+ ]
525
+
526
+ 48.times do |index|
527
+ suffix = index.to_s.rjust(2, "0")
528
+ warehouse_defs << {
529
+ key: "extract_#{suffix}", job: "WarehouseExtractJob",
530
+ start_after: 35 + ((index % 8) * 2), duration: 20 + (index % 5)
531
+ }
532
+ end
533
+
534
+ 48.times do |index|
535
+ suffix = index.to_s.rjust(2, "0")
536
+ attributes = {
537
+ key: "transform_#{suffix}", job: "WarehouseTransformJob",
538
+ start_after: 90 + ((index % 8) * 3), duration: 30 + (index % 7)
539
+ }
540
+ if index < 36
541
+ warehouse_defs << attributes
542
+ elsif index < 39
543
+ warehouse_defs << attributes.merge(status: "enqueued", job_state: :running)
544
+ elsif index < 42
545
+ warehouse_defs << attributes.merge(status: "enqueued", job_state: :queued)
546
+ else
547
+ warehouse_defs << attributes.merge(status: "pending", job_state: :none)
548
+ end
549
+ end
550
+
551
+ warehouse_defs << {
552
+ key: "merge", job: "WarehouseMergeJob", status: "pending", job_state: :none,
553
+ start_after: 160, duration: 15
554
+ }
555
+ 40.times do |index|
556
+ warehouse_defs << {
557
+ key: "load_#{index.to_s.rjust(2, '0')}", job: "WarehouseLoadJob",
558
+ status: "pending", job_state: :none, start_after: 180 + index, duration: 10
559
+ }
560
+ end
561
+ warehouse_defs << {
562
+ key: "finalize", job: "WarehouseFinalizeJob", status: "pending", job_state: :none,
563
+ start_after: 230, duration: 10
564
+ }
565
+
566
+ warehouse_steps = add_steps(warehouse, *warehouse_defs)
567
+ warehouse_edges = []
568
+ 48.times do |index|
569
+ suffix = index.to_s.rjust(2, "0")
570
+ warehouse_edges << ["plan", "extract_#{suffix}"]
571
+ warehouse_edges << ["extract_#{suffix}", "transform_#{suffix}"]
572
+ warehouse_edges << ["transform_#{suffix}", "merge"]
573
+ end
574
+ 40.times do |index|
575
+ suffix = index.to_s.rjust(2, "0")
576
+ warehouse_edges << ["merge", "load_#{suffix}"]
577
+ warehouse_edges << ["load_#{suffix}", "finalize"]
578
+ end
579
+ add_edges(warehouse, warehouse_steps, *warehouse_edges)
580
+
581
+ # Fixture B: 46 nodes connected to every subsequent node, producing exactly
582
+ # C(46, 2) = 1,035 edges while remaining below the 60-step large-pipeline gate.
583
+ dense = create_pipeline(
584
+ type: "DenseFanoutPipeline",
585
+ status: "succeeded",
586
+ params: { nodes: 46, topology: "fully_connected_forward" },
587
+ strategy: "ignore",
588
+ age: 26.hours,
589
+ duration: 12.minutes
590
+ )
591
+ dense_defs = 46.times.map do |index|
592
+ {
593
+ key: "node_#{index.to_s.rjust(2, '0')}", job: "DenseFanoutNodeJob",
594
+ start_after: 5 + (index * 10), duration: 5
595
+ }
596
+ end
597
+ dense_steps = add_steps(dense, *dense_defs)
598
+ dense_edges = 46.times.flat_map do |from|
599
+ ((from + 1)...46).map do |to|
600
+ ["node_#{from.to_s.rjust(2, '0')}", "node_#{to.to_s.rjust(2, '0')}"]
601
+ end
602
+ end
603
+ add_edges(dense, dense_steps, *dense_edges)
604
+
605
+ # A non-terminal execution is intentionally older than GoodJob's normal
606
+ # retention window. Its dangling IDs model job rows that cleanup has removed.
607
+ retention_gap = create_pipeline(
608
+ type: "LongRunningArchivePipeline",
609
+ status: "running",
610
+ params: { archive: "legal-hold", partitions: 4 },
611
+ strategy: "continue",
612
+ age: 20.days
613
+ )
614
+ retention_steps = add_steps(
615
+ retention_gap,
616
+ { key: "scan", job: "ArchiveScanJob", job_state: :dangling },
617
+ { key: "compact", job: "ArchiveCompactJob", job_state: :dangling },
618
+ { key: "verify", job: "ArchiveVerifyJob", status: "enqueued", job_state: :dangling },
619
+ { key: "publish", job: "ArchivePublishJob", status: "pending", job_state: :dangling }
620
+ )
621
+ add_edges(retention_gap, retention_steps, %w[scan compact], %w[compact verify], %w[verify publish])
622
+
623
+ # Add 180 compact historical executions. Together with the original 20 and the
624
+ # three scale/retention fixtures above, the dashboard has 203 rows to paginate.
625
+ # The generated set is exactly 80/8/5/4/3 percent (rounded) across statuses.
626
+ history_statuses = (["succeeded"] * 144) + (["running"] * 14) + (["failed"] * 9) +
627
+ (["halted"] * 7) + (["skipped"] * 6)
628
+ history_types = %w[
629
+ CatalogRefreshPipeline CustomerImportPipeline DailyMetricsPipeline
630
+ InventorySnapshotPipeline SearchReindexPipeline WebhookReplayPipeline
631
+ ]
632
+
633
+ 180.times do |index|
634
+ status = history_statuses[(index * 73) % history_statuses.length]
635
+ day_offset = index % 14
636
+ slot = index / 14
637
+ duration = status == "running" ? nil : (45 + ((index * 17) % 480)).seconds
638
+ pipeline = create_pipeline(
639
+ type: history_types[index % history_types.length],
640
+ status: status,
641
+ params: { run: index + 1, shard: index % 12, source: "demo" },
642
+ strategy: %w[halt continue ignore][index % 3],
643
+ created_at: recent_seed_time(day_offset, slot),
644
+ duration: duration,
645
+ halt_triggered: status == "halted"
646
+ )
647
+
648
+ step_defs = case status
649
+ when "succeeded"
650
+ [
651
+ { key: "prepare", job: "DemoPrepareJob", start_after: 5, duration: 8 },
652
+ { key: "process", job: "DemoProcessJob", start_after: 18, duration: 12 },
653
+ { key: "commit", job: "DemoCommitJob", start_after: 35, duration: 7 }
654
+ ]
655
+ when "running"
656
+ active_state = index.even? ? :running : :queued
657
+ [
658
+ { key: "prepare", job: "DemoPrepareJob", start_after: 5, duration: 8 },
659
+ { key: "process", job: "DemoProcessJob", status: "enqueued",
660
+ job_state: active_state, start_after: 18, duration: 12 },
661
+ { key: "commit", job: "DemoCommitJob", status: "pending", job_state: :none }
662
+ ]
663
+ when "failed"
664
+ [
665
+ { key: "prepare", job: "DemoPrepareJob", start_after: 5, duration: 8 },
666
+ { key: "process", job: "DemoProcessJob", status: "failed", start_after: 18, duration: 12,
667
+ error_class: "DemoImportError", error_message: "Synthetic row #{index + 1} could not be imported" },
668
+ { key: "commit", job: "DemoCommitJob", status: "skipped", job_state: :none }
669
+ ]
670
+ when "halted"
671
+ [
672
+ { key: "prepare", job: "DemoPrepareJob", status: "failed", start_after: 5, duration: 8,
673
+ error_class: "DemoHalt", error_message: "Synthetic safety check halted the execution" },
674
+ { key: "process", job: "DemoProcessJob", status: "halted", start_after: 18, duration: 4 },
675
+ { key: "commit", job: "DemoCommitJob", status: "skipped", job_state: :none }
676
+ ]
677
+ else
678
+ [
679
+ { key: "prepare", job: "DemoPrepareJob", status: "skipped", job_state: :none },
680
+ { key: "process", job: "DemoProcessJob", status: "skipped", job_state: :none },
681
+ { key: "commit", job: "DemoCommitJob", status: "skipped", job_state: :none }
682
+ ]
683
+ end
684
+
685
+ steps = add_steps(pipeline, *step_defs)
686
+ add_edges(pipeline, steps, %w[prepare process], %w[process commit])
687
+ end
688
+
689
+ puts "Done! #{GoodPipeline::PipelineRecord.count} pipelines, " \
690
+ "#{GoodPipeline::StepRecord.count} steps, and " \
691
+ "#{GoodJob::Job.where(queue_name: SEED_JOB_QUEUE).count} synthetic jobs seeded."
Binary file
Binary file