good_pipeline 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +33 -0
- data/README.md +52 -7
- data/app/controllers/good_pipeline/pipelines_controller.rb +3 -2
- data/app/frontend/good_pipeline/style.css +83 -4
- data/app/helpers/good_pipeline/mermaid_diagram_builder.rb +105 -0
- data/app/helpers/good_pipeline/pipelines_helper.rb +5 -30
- data/app/jobs/good_pipeline/pipeline_callback_job.rb +4 -4
- data/app/models/good_pipeline/pipeline_record.rb +1 -1
- data/app/models/good_pipeline/step_record.rb +9 -3
- data/app/views/good_pipeline/pipelines/definitions.html.erb +13 -1
- data/app/views/good_pipeline/pipelines/show.html.erb +13 -1
- data/app/views/layouts/good_pipeline/application.html.erb +115 -0
- data/demo/app/pipelines/branch_test_pipeline.rb +28 -0
- data/demo/db/migrate/20260319205326_create_good_pipeline_tables.rb +6 -2
- data/demo/db/seeds.rb +225 -1
- data/demo/docs/screenshots/definitions.png +0 -0
- data/demo/docs/screenshots/show.png +0 -0
- data/demo/test/good_pipeline/test_coordinator.rb +18 -0
- data/demo/test/good_pipeline/test_step_record.rb +12 -10
- data/demo/test/integration/test_branch_execution.rb +98 -0
- data/demo/test/integration/test_halt_ignore_chain.rb +75 -0
- data/demo/test/integration/test_ignore_transitive_exemption.rb +94 -0
- data/demo/test/integration/test_late_chain_registration.rb +80 -0
- data/demo/test/integration/test_missing_decision_method.rb +34 -0
- data/demo/test/integration/test_sequential_branches.rb +124 -0
- data/demo/test/integration/test_undeclared_branch_arm.rb +143 -0
- data/docs/.vitepress/config.mts +1 -0
- data/docs/architecture.md +14 -16
- data/docs/branching.md +135 -0
- data/docs/callbacks.md +3 -7
- data/docs/cleanup.md +2 -6
- data/docs/dag-validation.md +1 -1
- data/docs/dashboard.md +2 -2
- data/docs/defining-pipelines.md +25 -8
- data/docs/failure-strategies.md +4 -4
- data/docs/getting-started.md +2 -1
- data/docs/index.md +7 -7
- data/docs/introduction.md +12 -12
- data/docs/monitoring.md +20 -20
- data/docs/pipeline-chaining.md +6 -5
- data/docs/public/screenshots/definitions.png +0 -0
- data/docs/public/screenshots/index.png +0 -0
- data/docs/public/screenshots/show.png +0 -0
- data/docs/screenshots/definitions.png +0 -0
- data/docs/screenshots/index.png +0 -0
- data/docs/screenshots/show.png +0 -0
- data/lib/generators/good_pipeline/install/templates/create_good_pipeline_tables.rb.erb +8 -4
- data/lib/good_pipeline/branch_builder.rb +23 -0
- data/lib/good_pipeline/branch_resolver.rb +55 -0
- data/lib/good_pipeline/chain.rb +8 -0
- data/lib/good_pipeline/chain_coordinator.rb +34 -33
- data/lib/good_pipeline/coordinator.rb +156 -133
- data/lib/good_pipeline/cycle_detector.rb +24 -22
- data/lib/good_pipeline/failure_metadata.rb +18 -16
- data/lib/good_pipeline/pipeline.rb +80 -10
- data/lib/good_pipeline/runner.rb +23 -4
- data/lib/good_pipeline/step_definition.rb +32 -4
- data/lib/good_pipeline/version.rb +1 -1
- data/lib/good_pipeline.rb +2 -0
- metadata +15 -1
|
@@ -36,5 +36,120 @@
|
|
|
36
36
|
});
|
|
37
37
|
});
|
|
38
38
|
</script>
|
|
39
|
+
<script>
|
|
40
|
+
(function() {
|
|
41
|
+
var MIN_SCALE = 0.5, MAX_SCALE = 4, ZOOM_STEP = 0.25;
|
|
42
|
+
var states = new WeakMap();
|
|
43
|
+
|
|
44
|
+
function getState(container) {
|
|
45
|
+
if (!states.has(container)) states.set(container, { scale: 1, panX: 0, panY: 0 });
|
|
46
|
+
return states.get(container);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function applyTransform(container) {
|
|
50
|
+
var s = getState(container);
|
|
51
|
+
var canvas = container.querySelector('.diagram-canvas');
|
|
52
|
+
if (canvas) canvas.style.transform = 'translate(' + s.panX + 'px, ' + s.panY + 'px) scale(' + s.scale + ')';
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function clampScale(val) {
|
|
56
|
+
return Math.min(MAX_SCALE, Math.max(MIN_SCALE, val));
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function zoomAroundPoint(s, centerX, centerY, newScale) {
|
|
60
|
+
var oldScale = s.scale;
|
|
61
|
+
newScale = clampScale(newScale);
|
|
62
|
+
if (newScale === oldScale) return;
|
|
63
|
+
var ratio = newScale / oldScale;
|
|
64
|
+
s.panX = centerX - (centerX - s.panX) * ratio;
|
|
65
|
+
s.panY = centerY - (centerY - s.panY) * ratio;
|
|
66
|
+
s.scale = newScale;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Button actions via event delegation
|
|
70
|
+
document.addEventListener('click', function(e) {
|
|
71
|
+
var btn = e.target.closest('[data-action]');
|
|
72
|
+
if (!btn) return;
|
|
73
|
+
var container = btn.closest('.diagram-container');
|
|
74
|
+
if (!container) return;
|
|
75
|
+
var s = getState(container);
|
|
76
|
+
var action = btn.dataset.action;
|
|
77
|
+
|
|
78
|
+
if (action === 'zoom-in' || action === 'zoom-out') {
|
|
79
|
+
var viewport = container.querySelector('.diagram-viewport');
|
|
80
|
+
var rect = viewport.getBoundingClientRect();
|
|
81
|
+
var centerX = rect.width / 2;
|
|
82
|
+
var centerY = rect.height / 2;
|
|
83
|
+
var delta = action === 'zoom-in' ? ZOOM_STEP : -ZOOM_STEP;
|
|
84
|
+
zoomAroundPoint(s, centerX, centerY, s.scale + delta);
|
|
85
|
+
} else if (action === 'zoom-reset') {
|
|
86
|
+
s.scale = 1; s.panX = 0; s.panY = 0;
|
|
87
|
+
} else if (action === 'fullscreen') {
|
|
88
|
+
if (document.fullscreenElement === container) {
|
|
89
|
+
document.exitFullscreen();
|
|
90
|
+
} else {
|
|
91
|
+
container.requestFullscreen();
|
|
92
|
+
}
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
applyTransform(container);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
// Mouse wheel zoom toward cursor
|
|
99
|
+
document.querySelectorAll('.diagram-viewport').forEach(function(viewport) {
|
|
100
|
+
viewport.addEventListener('wheel', function(e) {
|
|
101
|
+
e.preventDefault();
|
|
102
|
+
var container = viewport.closest('.diagram-container');
|
|
103
|
+
var s = getState(container);
|
|
104
|
+
var rect = viewport.getBoundingClientRect();
|
|
105
|
+
var cursorX = e.clientX - rect.left;
|
|
106
|
+
var cursorY = e.clientY - rect.top;
|
|
107
|
+
var delta = e.deltaY < 0 ? ZOOM_STEP : -ZOOM_STEP;
|
|
108
|
+
zoomAroundPoint(s, cursorX, cursorY, s.scale + delta);
|
|
109
|
+
applyTransform(container);
|
|
110
|
+
}, { passive: false });
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
// Click-drag pan
|
|
114
|
+
document.querySelectorAll('.diagram-viewport').forEach(function(viewport) {
|
|
115
|
+
viewport.addEventListener('mousedown', function(e) {
|
|
116
|
+
if (e.button !== 0) return;
|
|
117
|
+
var container = viewport.closest('.diagram-container');
|
|
118
|
+
var canvas = container.querySelector('.diagram-canvas');
|
|
119
|
+
var s = getState(container);
|
|
120
|
+
var startX = e.clientX - s.panX;
|
|
121
|
+
var startY = e.clientY - s.panY;
|
|
122
|
+
canvas.classList.add('dragging');
|
|
123
|
+
|
|
124
|
+
function onMove(ev) {
|
|
125
|
+
s.panX = ev.clientX - startX;
|
|
126
|
+
s.panY = ev.clientY - startY;
|
|
127
|
+
applyTransform(container);
|
|
128
|
+
}
|
|
129
|
+
function onUp() {
|
|
130
|
+
canvas.classList.remove('dragging');
|
|
131
|
+
document.removeEventListener('mousemove', onMove);
|
|
132
|
+
document.removeEventListener('mouseup', onUp);
|
|
133
|
+
}
|
|
134
|
+
document.addEventListener('mousemove', onMove);
|
|
135
|
+
document.addEventListener('mouseup', onUp);
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
// Reset zoom when switching definitions
|
|
140
|
+
document.querySelectorAll('.definition-item').forEach(function(btn) {
|
|
141
|
+
btn.addEventListener('click', function() {
|
|
142
|
+
var type = btn.dataset.pipeline;
|
|
143
|
+
var panel = document.querySelector('.definition-panel[data-pipeline="' + type + '"]');
|
|
144
|
+
if (!panel) return;
|
|
145
|
+
var container = panel.querySelector('.diagram-container');
|
|
146
|
+
if (!container) return;
|
|
147
|
+
var s = getState(container);
|
|
148
|
+
s.scale = 1; s.panX = 0; s.panY = 0;
|
|
149
|
+
applyTransform(container);
|
|
150
|
+
});
|
|
151
|
+
});
|
|
152
|
+
})();
|
|
153
|
+
</script>
|
|
39
154
|
</body>
|
|
40
155
|
</html>
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class BranchTestPipeline < GoodPipeline::Pipeline
|
|
4
|
+
description "Pipeline with branching for integration tests"
|
|
5
|
+
|
|
6
|
+
def configure(choice:, **)
|
|
7
|
+
run :analyze, DownloadJob, with: { choice: choice }
|
|
8
|
+
|
|
9
|
+
branch :format_check, after: :analyze, by: :pick_format do
|
|
10
|
+
on :hd do
|
|
11
|
+
run :transcode_hd, TranscodeJob, with: { choice: choice }
|
|
12
|
+
run :upscale, ThumbnailJob, with: { choice: choice }, after: :transcode_hd
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
on :sd do
|
|
16
|
+
run :transcode_sd, PublishJob, with: { choice: choice }
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
run :finish, CleanupJob, with: { choice: choice }, after: :format_check
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
def pick_format
|
|
26
|
+
params[:choice].to_sym
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -26,8 +26,8 @@ class CreateGoodPipelineTables < ActiveRecord::Migration[8.1]
|
|
|
26
26
|
t.jsonb :params, null: false, default: {}
|
|
27
27
|
t.string :coordination_status, null: false, default: "pending"
|
|
28
28
|
t.string :on_failure_strategy
|
|
29
|
-
t.
|
|
30
|
-
t.
|
|
29
|
+
t.jsonb :enqueue_options, null: false, default: {}
|
|
30
|
+
t.jsonb :branch, null: false, default: {}
|
|
31
31
|
t.uuid :good_job_batch_id
|
|
32
32
|
t.uuid :good_job_id
|
|
33
33
|
t.integer :attempts
|
|
@@ -38,6 +38,7 @@ class CreateGoodPipelineTables < ActiveRecord::Migration[8.1]
|
|
|
38
38
|
end
|
|
39
39
|
|
|
40
40
|
add_index :good_pipeline_steps, %i[pipeline_id key], unique: true
|
|
41
|
+
add_index :good_pipeline_steps, :coordination_status
|
|
41
42
|
|
|
42
43
|
create_table :good_pipeline_dependencies do |t|
|
|
43
44
|
t.references :pipeline, null: false, foreign_key: { to_table: :good_pipeline_pipelines }, type: :uuid
|
|
@@ -49,5 +50,8 @@ class CreateGoodPipelineTables < ActiveRecord::Migration[8.1]
|
|
|
49
50
|
t.references :upstream_pipeline, null: false, foreign_key: { to_table: :good_pipeline_pipelines }, type: :uuid
|
|
50
51
|
t.references :downstream_pipeline, null: false, foreign_key: { to_table: :good_pipeline_pipelines }, type: :uuid
|
|
51
52
|
end
|
|
53
|
+
|
|
54
|
+
add_index :good_pipeline_chains, %i[upstream_pipeline_id downstream_pipeline_id], unique: true,
|
|
55
|
+
name: :index_good_pipeline_chains_uniqueness
|
|
52
56
|
end
|
|
53
57
|
end
|
data/demo/db/seeds.rb
CHANGED
|
@@ -24,10 +24,13 @@ end
|
|
|
24
24
|
def add_steps(pipeline, *step_defs)
|
|
25
25
|
records = {}
|
|
26
26
|
step_defs.each do |definition|
|
|
27
|
+
branch_hash = definition.slice(:decides, :branch_result, :empty_arms, :branch_key, :branch_arm).transform_keys(&:to_s)
|
|
28
|
+
|
|
27
29
|
records[definition[:key]] = GoodPipeline::StepRecord.create!(
|
|
28
30
|
pipeline: pipeline, key: definition[:key], job_class: definition[:job],
|
|
29
31
|
coordination_status: definition.fetch(:status, "succeeded"),
|
|
30
|
-
error_class: definition[:error_class], error_message: definition[:error_message]
|
|
32
|
+
error_class: definition[:error_class], error_message: definition[:error_message],
|
|
33
|
+
branch: branch_hash
|
|
31
34
|
)
|
|
32
35
|
end
|
|
33
36
|
records
|
|
@@ -150,4 +153,225 @@ ingest2_steps = add_steps(ingest2,
|
|
|
150
153
|
{ key: "load", job: "LoadJob", status: "pending" })
|
|
151
154
|
add_edges(ingest2, ingest2_steps, %w[extract validate], %w[validate transform], %w[transform load])
|
|
152
155
|
|
|
156
|
+
# 11. MediaProcessingPipeline (succeeded, HD path taken) — single branch
|
|
157
|
+
branch_job = GoodPipeline::BRANCH_JOB_CLASS
|
|
158
|
+
media = create_pipeline(type: "MediaProcessingPipeline", status: "succeeded",
|
|
159
|
+
params: { media_id: 4421, source: "upload" }, age: 1.hour, duration: 20.minutes)
|
|
160
|
+
media_steps = add_steps(media,
|
|
161
|
+
{ key: "analyze", job: "AnalyzeJob" },
|
|
162
|
+
{ key: "format_check", job: branch_job, decides: "detect_format", branch_result: "hd" },
|
|
163
|
+
{ key: "transcode_hd", job: "TranscodeHDJob", branch_key: "format_check", branch_arm: "hd" },
|
|
164
|
+
{ key: "upscale", job: "UpscaleJob", branch_key: "format_check", branch_arm: "hd" },
|
|
165
|
+
{ key: "transcode_sd", job: "TranscodeSDJob", branch_key: "format_check", branch_arm: "sd",
|
|
166
|
+
status: "skipped_by_branch" },
|
|
167
|
+
{ key: "publish", job: "PublishJob" })
|
|
168
|
+
add_edges(media, media_steps,
|
|
169
|
+
%w[analyze format_check],
|
|
170
|
+
%w[format_check transcode_hd], %w[format_check transcode_sd],
|
|
171
|
+
%w[transcode_hd upscale],
|
|
172
|
+
%w[upscale publish], %w[transcode_sd publish])
|
|
173
|
+
|
|
174
|
+
# 12. DeploymentPipeline (succeeded, canary path) — branch with multi-step arms
|
|
175
|
+
deploy = create_pipeline(type: "DeploymentPipeline", status: "succeeded",
|
|
176
|
+
params: { service: "api-gateway", version: "2.4.1", env: "production" },
|
|
177
|
+
age: 4.hours, duration: 45.minutes)
|
|
178
|
+
deploy_steps = add_steps(deploy,
|
|
179
|
+
{ key: "build", job: "BuildJob" },
|
|
180
|
+
{ key: "run_tests", job: "RunTestsJob" },
|
|
181
|
+
{ key: "deploy_strategy", job: branch_job, decides: "pick_strategy", branch_result: "canary" },
|
|
182
|
+
{ key: "canary_deploy", job: "CanaryDeployJob", branch_key: "deploy_strategy", branch_arm: "canary" },
|
|
183
|
+
{ key: "canary_monitor", job: "CanaryMonitorJob", branch_key: "deploy_strategy", branch_arm: "canary" },
|
|
184
|
+
{ key: "canary_promote", job: "CanaryPromoteJob", branch_key: "deploy_strategy", branch_arm: "canary" },
|
|
185
|
+
{ key: "blue_green_swap", job: "BlueGreenSwapJob", branch_key: "deploy_strategy", branch_arm: "blue_green",
|
|
186
|
+
status: "skipped_by_branch" },
|
|
187
|
+
{ key: "blue_green_verify", job: "BlueGreenVerifyJob", branch_key: "deploy_strategy", branch_arm: "blue_green",
|
|
188
|
+
status: "skipped_by_branch" },
|
|
189
|
+
{ key: "notify_team", job: "NotifyTeamJob" })
|
|
190
|
+
add_edges(deploy, deploy_steps,
|
|
191
|
+
%w[build run_tests], %w[run_tests deploy_strategy],
|
|
192
|
+
%w[deploy_strategy canary_deploy], %w[deploy_strategy blue_green_swap],
|
|
193
|
+
%w[canary_deploy canary_monitor], %w[canary_monitor canary_promote],
|
|
194
|
+
%w[blue_green_swap blue_green_verify],
|
|
195
|
+
%w[canary_promote notify_team], %w[blue_green_verify notify_team])
|
|
196
|
+
|
|
197
|
+
# 13. MediaProcessingPipeline (succeeded, SD path) — same pipeline type, different branch result
|
|
198
|
+
media2 = create_pipeline(type: "MediaProcessingPipeline", status: "succeeded",
|
|
199
|
+
params: { media_id: 4422, source: "api" }, age: 30.minutes, duration: 10.minutes)
|
|
200
|
+
media2_steps = add_steps(media2,
|
|
201
|
+
{ key: "analyze", job: "AnalyzeJob" },
|
|
202
|
+
{ key: "format_check", job: branch_job, decides: "detect_format", branch_result: "sd" },
|
|
203
|
+
{ key: "transcode_hd", job: "TranscodeHDJob", branch_key: "format_check", branch_arm: "hd",
|
|
204
|
+
status: "skipped_by_branch" },
|
|
205
|
+
{ key: "upscale", job: "UpscaleJob", branch_key: "format_check", branch_arm: "hd",
|
|
206
|
+
status: "skipped_by_branch" },
|
|
207
|
+
{ key: "transcode_sd", job: "TranscodeSDJob", branch_key: "format_check", branch_arm: "sd" },
|
|
208
|
+
{ key: "publish", job: "PublishJob" })
|
|
209
|
+
add_edges(media2, media2_steps,
|
|
210
|
+
%w[analyze format_check],
|
|
211
|
+
%w[format_check transcode_hd], %w[format_check transcode_sd],
|
|
212
|
+
%w[transcode_hd upscale],
|
|
213
|
+
%w[upscale publish], %w[transcode_sd publish])
|
|
214
|
+
|
|
215
|
+
# 14. ContentModerationPipeline (succeeded) — two sequential branches
|
|
216
|
+
# ingest → classify_content(branch: text/image) → [text arm / image arm] →
|
|
217
|
+
# review_priority(branch: high/low) → [fast review / standard review] → publish
|
|
218
|
+
moderation = create_pipeline(type: "ContentModerationPipeline", status: "succeeded",
|
|
219
|
+
params: { content_id: 99_201, source: "user_upload" },
|
|
220
|
+
age: 2.hours, duration: 35.minutes)
|
|
221
|
+
moderation_steps = add_steps(moderation,
|
|
222
|
+
{ key: "ingest", job: "IngestContentJob" },
|
|
223
|
+
# First branch: content type
|
|
224
|
+
{ key: "classify_content", job: branch_job, decides: "detect_content_type",
|
|
225
|
+
branch_result: "image" },
|
|
226
|
+
{ key: "extract_text", job: "ExtractTextJob",
|
|
227
|
+
branch_key: "classify_content", branch_arm: "text", status: "skipped_by_branch" },
|
|
228
|
+
{ key: "run_nlp", job: "RunNlpJob",
|
|
229
|
+
branch_key: "classify_content", branch_arm: "text", status: "skipped_by_branch" },
|
|
230
|
+
{ key: "detect_objects", job: "DetectObjectsJob",
|
|
231
|
+
branch_key: "classify_content", branch_arm: "image" },
|
|
232
|
+
{ key: "check_nsfw", job: "CheckNsfwJob",
|
|
233
|
+
branch_key: "classify_content", branch_arm: "image" },
|
|
234
|
+
# Second branch: review priority
|
|
235
|
+
{ key: "review_priority", job: branch_job, decides: "determine_priority",
|
|
236
|
+
branch_result: "low" },
|
|
237
|
+
{ key: "fast_review", job: "FastReviewJob",
|
|
238
|
+
branch_key: "review_priority", branch_arm: "high", status: "skipped_by_branch" },
|
|
239
|
+
{ key: "standard_review", job: "StandardReviewJob",
|
|
240
|
+
branch_key: "review_priority", branch_arm: "low" },
|
|
241
|
+
{ key: "publish_content", job: "PublishContentJob" })
|
|
242
|
+
add_edges(moderation, moderation_steps,
|
|
243
|
+
%w[ingest classify_content],
|
|
244
|
+
%w[classify_content extract_text], %w[classify_content detect_objects],
|
|
245
|
+
%w[extract_text run_nlp],
|
|
246
|
+
%w[detect_objects check_nsfw],
|
|
247
|
+
# Both arms feed into second branch
|
|
248
|
+
%w[run_nlp review_priority], %w[check_nsfw review_priority],
|
|
249
|
+
%w[review_priority fast_review], %w[review_priority standard_review],
|
|
250
|
+
%w[fast_review publish_content], %w[standard_review publish_content])
|
|
251
|
+
|
|
252
|
+
# 15. DataEnrichmentPipeline (succeeded, skip arm taken) — branch with empty arm
|
|
253
|
+
enrichment = create_pipeline(type: "DataEnrichmentPipeline", status: "succeeded",
|
|
254
|
+
params: { record_id: 33_100, source: "api" },
|
|
255
|
+
age: 20.minutes, duration: 8.minutes)
|
|
256
|
+
enrichment_steps = add_steps(enrichment,
|
|
257
|
+
{ key: "fetch_record", job: "FetchRecordJob" },
|
|
258
|
+
{ key: "quality_check", job: branch_job, decides: "needs_enrichment",
|
|
259
|
+
branch_result: "skip", empty_arms: %w[skip] },
|
|
260
|
+
{ key: "geocode", job: "GeocodeJob",
|
|
261
|
+
branch_key: "quality_check", branch_arm: "enrich", status: "skipped_by_branch" },
|
|
262
|
+
{ key: "normalize", job: "NormalizeJob",
|
|
263
|
+
branch_key: "quality_check", branch_arm: "enrich", status: "skipped_by_branch" },
|
|
264
|
+
{ key: "save_record", job: "SaveRecordJob" })
|
|
265
|
+
add_edges(enrichment, enrichment_steps,
|
|
266
|
+
%w[fetch_record quality_check],
|
|
267
|
+
%w[quality_check geocode],
|
|
268
|
+
%w[geocode normalize],
|
|
269
|
+
%w[normalize save_record])
|
|
270
|
+
|
|
271
|
+
# 16. DataEnrichmentPipeline (succeeded, enrich arm taken) — same type, different path
|
|
272
|
+
enrichment2 = create_pipeline(type: "DataEnrichmentPipeline", status: "succeeded",
|
|
273
|
+
params: { record_id: 33_101, source: "upload" },
|
|
274
|
+
age: 10.minutes, duration: 15.minutes)
|
|
275
|
+
enrichment2_steps = add_steps(enrichment2,
|
|
276
|
+
{ key: "fetch_record", job: "FetchRecordJob" },
|
|
277
|
+
{ key: "quality_check", job: branch_job, decides: "needs_enrichment",
|
|
278
|
+
branch_result: "enrich", empty_arms: %w[skip] },
|
|
279
|
+
{ key: "geocode", job: "GeocodeJob",
|
|
280
|
+
branch_key: "quality_check", branch_arm: "enrich" },
|
|
281
|
+
{ key: "normalize", job: "NormalizeJob",
|
|
282
|
+
branch_key: "quality_check", branch_arm: "enrich" },
|
|
283
|
+
{ key: "save_record", job: "SaveRecordJob" })
|
|
284
|
+
add_edges(enrichment2, enrichment2_steps,
|
|
285
|
+
%w[fetch_record quality_check],
|
|
286
|
+
%w[quality_check geocode],
|
|
287
|
+
%w[geocode normalize],
|
|
288
|
+
%w[normalize save_record])
|
|
289
|
+
|
|
290
|
+
# 17. OrderRoutingPipeline (succeeded, skip on first branch, process on second) — 2 sequential branches with empty arms
|
|
291
|
+
routing = create_pipeline(type: "OrderRoutingPipeline", status: "succeeded",
|
|
292
|
+
params: { order_id: 50_100, region: "eu" },
|
|
293
|
+
age: 40.minutes, duration: 12.minutes)
|
|
294
|
+
routing_steps = add_steps(routing,
|
|
295
|
+
{ key: "receive_order", job: "ReceiveOrderJob" },
|
|
296
|
+
# First branch: fraud check — skip (no fraud)
|
|
297
|
+
{ key: "fraud_check", job: branch_job, decides: "check_fraud",
|
|
298
|
+
branch_result: "safe", empty_arms: %w[safe] },
|
|
299
|
+
{ key: "manual_review", job: "ManualReviewJob",
|
|
300
|
+
branch_key: "fraud_check", branch_arm: "suspicious",
|
|
301
|
+
status: "skipped_by_branch" },
|
|
302
|
+
{ key: "flag_account", job: "FlagAccountJob",
|
|
303
|
+
branch_key: "fraud_check", branch_arm: "suspicious",
|
|
304
|
+
status: "skipped_by_branch" },
|
|
305
|
+
# Second branch: shipping method
|
|
306
|
+
{ key: "shipping_method", job: branch_job, decides: "pick_shipping",
|
|
307
|
+
branch_result: "express", empty_arms: %w[pickup] },
|
|
308
|
+
{ key: "schedule_courier", job: "ScheduleCourierJob",
|
|
309
|
+
branch_key: "shipping_method", branch_arm: "express" },
|
|
310
|
+
{ key: "generate_label", job: "GenerateLabelJob",
|
|
311
|
+
branch_key: "shipping_method", branch_arm: "express" },
|
|
312
|
+
{ key: "notify_warehouse", job: "NotifyWarehouseJob",
|
|
313
|
+
branch_key: "shipping_method", branch_arm: "standard",
|
|
314
|
+
status: "skipped_by_branch" },
|
|
315
|
+
{ key: "send_confirmation", job: "SendConfirmationJob" })
|
|
316
|
+
add_edges(routing, routing_steps,
|
|
317
|
+
%w[receive_order fraud_check],
|
|
318
|
+
%w[fraud_check manual_review], %w[manual_review flag_account],
|
|
319
|
+
# First branch exit → second branch
|
|
320
|
+
%w[flag_account shipping_method],
|
|
321
|
+
%w[shipping_method schedule_courier], %w[shipping_method notify_warehouse],
|
|
322
|
+
%w[schedule_courier generate_label],
|
|
323
|
+
%w[generate_label send_confirmation], %w[notify_warehouse send_confirmation])
|
|
324
|
+
|
|
325
|
+
# 18. OrderRoutingPipeline (succeeded, suspicious + pickup) — both empty arms skipped through
|
|
326
|
+
routing2 = create_pipeline(type: "OrderRoutingPipeline", status: "succeeded",
|
|
327
|
+
params: { order_id: 50_101, region: "us" },
|
|
328
|
+
age: 25.minutes, duration: 18.minutes)
|
|
329
|
+
routing2_steps = add_steps(routing2,
|
|
330
|
+
{ key: "receive_order", job: "ReceiveOrderJob" },
|
|
331
|
+
# First branch: fraud — suspicious path taken
|
|
332
|
+
{ key: "fraud_check", job: branch_job, decides: "check_fraud",
|
|
333
|
+
branch_result: "suspicious", empty_arms: %w[safe] },
|
|
334
|
+
{ key: "manual_review", job: "ManualReviewJob",
|
|
335
|
+
branch_key: "fraud_check", branch_arm: "suspicious" },
|
|
336
|
+
{ key: "flag_account", job: "FlagAccountJob",
|
|
337
|
+
branch_key: "fraud_check", branch_arm: "suspicious" },
|
|
338
|
+
# Second branch: shipping — pickup (empty arm)
|
|
339
|
+
{ key: "shipping_method", job: branch_job, decides: "pick_shipping",
|
|
340
|
+
branch_result: "pickup", empty_arms: %w[pickup] },
|
|
341
|
+
{ key: "schedule_courier", job: "ScheduleCourierJob",
|
|
342
|
+
branch_key: "shipping_method", branch_arm: "express",
|
|
343
|
+
status: "skipped_by_branch" },
|
|
344
|
+
{ key: "generate_label", job: "GenerateLabelJob",
|
|
345
|
+
branch_key: "shipping_method", branch_arm: "express",
|
|
346
|
+
status: "skipped_by_branch" },
|
|
347
|
+
{ key: "notify_warehouse", job: "NotifyWarehouseJob",
|
|
348
|
+
branch_key: "shipping_method", branch_arm: "standard",
|
|
349
|
+
status: "skipped_by_branch" },
|
|
350
|
+
{ key: "send_confirmation", job: "SendConfirmationJob" })
|
|
351
|
+
add_edges(routing2, routing2_steps,
|
|
352
|
+
%w[receive_order fraud_check],
|
|
353
|
+
%w[fraud_check manual_review], %w[manual_review flag_account],
|
|
354
|
+
%w[flag_account shipping_method],
|
|
355
|
+
%w[shipping_method schedule_courier], %w[shipping_method notify_warehouse],
|
|
356
|
+
%w[schedule_courier generate_label],
|
|
357
|
+
%w[generate_label send_confirmation], %w[notify_warehouse send_confirmation])
|
|
358
|
+
|
|
359
|
+
# 19. NotificationRoutingPipeline (succeeded, skip arm to End) — branch at the end with empty arm
|
|
360
|
+
notif_routing = create_pipeline(type: "NotificationRoutingPipeline", status: "succeeded",
|
|
361
|
+
params: { user_id: 77_200, event: "purchase" },
|
|
362
|
+
age: 5.minutes, duration: 3.minutes)
|
|
363
|
+
notif_routing_steps = add_steps(notif_routing,
|
|
364
|
+
{ key: "load_preferences", job: "LoadPreferencesJob" },
|
|
365
|
+
{ key: "notification_channel", job: branch_job, decides: "preferred_channel",
|
|
366
|
+
branch_result: "none", empty_arms: %w[none] },
|
|
367
|
+
{ key: "send_email", job: "SendEmailJob",
|
|
368
|
+
branch_key: "notification_channel", branch_arm: "email",
|
|
369
|
+
status: "skipped_by_branch" },
|
|
370
|
+
{ key: "send_push", job: "SendPushJob",
|
|
371
|
+
branch_key: "notification_channel", branch_arm: "push",
|
|
372
|
+
status: "skipped_by_branch" })
|
|
373
|
+
add_edges(notif_routing, notif_routing_steps,
|
|
374
|
+
%w[load_preferences notification_channel],
|
|
375
|
+
%w[notification_channel send_email], %w[notification_channel send_push])
|
|
376
|
+
|
|
153
377
|
puts "Done! #{GoodPipeline::PipelineRecord.count} pipelines seeded."
|
|
Binary file
|
|
Binary file
|
|
@@ -193,6 +193,24 @@ class TestCoordinator < ActiveSupport::TestCase
|
|
|
193
193
|
assert_predicate pipeline.reload, :halt_triggered?
|
|
194
194
|
end
|
|
195
195
|
|
|
196
|
+
def test_halt_with_step_ignore_exempts_transitive_descendants
|
|
197
|
+
pipeline = create_pipeline(on_failure_strategy: "halt")
|
|
198
|
+
pipeline.update_columns(status: "running")
|
|
199
|
+
step_a = build_step(pipeline, key: "a", on_failure_strategy: "ignore")
|
|
200
|
+
step_b = build_step(pipeline, key: "b", dependencies: [step_a])
|
|
201
|
+
step_c = build_step(pipeline, key: "c", dependencies: [step_b])
|
|
202
|
+
step_d = build_step(pipeline, key: "d")
|
|
203
|
+
step_a.update_columns(coordination_status: "enqueued")
|
|
204
|
+
|
|
205
|
+
GoodPipeline::Coordinator.complete_step(step_a.reload, succeeded: false)
|
|
206
|
+
|
|
207
|
+
refute_equal "skipped", step_b.reload.coordination_status
|
|
208
|
+
refute_equal "skipped", step_c.reload.coordination_status,
|
|
209
|
+
"Transitive descendant of :ignore step should NOT be skipped by halt"
|
|
210
|
+
assert_equal "skipped", step_d.reload.coordination_status,
|
|
211
|
+
"Unrelated step should still be skipped under :halt"
|
|
212
|
+
end
|
|
213
|
+
|
|
196
214
|
# --- Continue strategy ---
|
|
197
215
|
|
|
198
216
|
def test_continue_skips_only_unsatisfied_descendants
|
|
@@ -100,20 +100,20 @@ class TestStepRecord < ActiveSupport::TestCase
|
|
|
100
100
|
|
|
101
101
|
# --- transition_coordination_status_to! invalid transitions ---
|
|
102
102
|
|
|
103
|
-
def
|
|
103
|
+
def test_transition_pending_to_succeeded_allowed_for_branch_steps
|
|
104
104
|
pipeline = create_pipeline
|
|
105
105
|
step = create_step(pipeline)
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
106
|
+
step.transition_coordination_status_to!(:succeeded)
|
|
107
|
+
|
|
108
|
+
assert_equal "succeeded", step.coordination_status
|
|
109
109
|
end
|
|
110
110
|
|
|
111
|
-
def
|
|
111
|
+
def test_transition_pending_to_failed_allowed_for_branch_resolution_failures
|
|
112
112
|
pipeline = create_pipeline
|
|
113
113
|
step = create_step(pipeline)
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
114
|
+
step.transition_coordination_status_to!(:failed)
|
|
115
|
+
|
|
116
|
+
assert_equal "failed", step.coordination_status
|
|
117
117
|
end
|
|
118
118
|
|
|
119
119
|
def test_transition_enqueued_to_pending_raises
|
|
@@ -169,11 +169,13 @@ class TestStepRecord < ActiveSupport::TestCase
|
|
|
169
169
|
def test_error_message_includes_step_key
|
|
170
170
|
pipeline = create_pipeline
|
|
171
171
|
step = create_step(pipeline, key: "transcode")
|
|
172
|
+
step.transition_coordination_status_to!(:enqueued)
|
|
172
173
|
error = assert_raises(GoodPipeline::InvalidTransition) do
|
|
173
|
-
step.transition_coordination_status_to!(:
|
|
174
|
+
step.transition_coordination_status_to!(:pending)
|
|
174
175
|
end
|
|
176
|
+
|
|
175
177
|
assert_includes error.message, "transcode"
|
|
176
|
-
assert_includes error.message, "from '
|
|
178
|
+
assert_includes error.message, "from 'enqueued' to 'pending'"
|
|
177
179
|
end
|
|
178
180
|
|
|
179
181
|
# --- Accepts symbols ---
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "test_helper"
|
|
4
|
+
|
|
5
|
+
class TestBranchExecution < ActiveSupport::TestCase
|
|
6
|
+
def test_full_branch_pipeline_hd_path
|
|
7
|
+
chain = BranchTestPipeline.run(choice: "hd")
|
|
8
|
+
perform_enqueued_jobs_inline
|
|
9
|
+
|
|
10
|
+
pipeline = chain.reload
|
|
11
|
+
assert_equal "succeeded", pipeline.status
|
|
12
|
+
|
|
13
|
+
steps = pipeline.steps.index_by(&:key)
|
|
14
|
+
assert_equal "succeeded", steps["analyze"].coordination_status
|
|
15
|
+
assert_equal "succeeded", steps["format_check"].coordination_status
|
|
16
|
+
assert_equal "succeeded", steps["transcode_hd"].coordination_status
|
|
17
|
+
assert_equal "succeeded", steps["upscale"].coordination_status
|
|
18
|
+
assert_equal "skipped_by_branch", steps["transcode_sd"].coordination_status
|
|
19
|
+
assert_equal "succeeded", steps["finish"].coordination_status
|
|
20
|
+
|
|
21
|
+
branch_step = steps["format_check"]
|
|
22
|
+
assert_equal "hd", branch_step.branch_result
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def test_full_branch_pipeline_sd_path
|
|
26
|
+
chain = BranchTestPipeline.run(choice: "sd")
|
|
27
|
+
perform_enqueued_jobs_inline
|
|
28
|
+
|
|
29
|
+
pipeline = chain.reload
|
|
30
|
+
steps = pipeline.steps.index_by(&:key)
|
|
31
|
+
|
|
32
|
+
assert_equal "succeeded", pipeline.status
|
|
33
|
+
assert_equal "succeeded", steps["analyze"].coordination_status
|
|
34
|
+
assert_equal "succeeded", steps["format_check"].coordination_status
|
|
35
|
+
assert_equal "skipped_by_branch", steps["transcode_hd"].coordination_status
|
|
36
|
+
assert_equal "skipped_by_branch", steps["upscale"].coordination_status
|
|
37
|
+
assert_equal "succeeded", steps["transcode_sd"].coordination_status
|
|
38
|
+
assert_equal "succeeded", steps["finish"].coordination_status
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def test_branch_skipped_arm_satisfies_downstream
|
|
42
|
+
chain = BranchTestPipeline.run(choice: "hd")
|
|
43
|
+
perform_enqueued_jobs_inline
|
|
44
|
+
|
|
45
|
+
pipeline = chain.reload
|
|
46
|
+
finish_step = pipeline.steps.find_by(key: "finish")
|
|
47
|
+
|
|
48
|
+
assert_equal "succeeded", finish_step.coordination_status
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def test_branch_with_multiple_steps_per_arm
|
|
52
|
+
chain = BranchTestPipeline.run(choice: "hd")
|
|
53
|
+
perform_enqueued_jobs_inline
|
|
54
|
+
|
|
55
|
+
pipeline = chain.reload
|
|
56
|
+
steps = pipeline.steps.index_by(&:key)
|
|
57
|
+
|
|
58
|
+
assert_equal "succeeded", steps["transcode_hd"].coordination_status
|
|
59
|
+
assert_equal "succeeded", steps["upscale"].coordination_status
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def test_branch_decision_result_cached_on_step
|
|
63
|
+
chain = BranchTestPipeline.run(choice: "sd")
|
|
64
|
+
perform_enqueued_jobs_inline
|
|
65
|
+
|
|
66
|
+
pipeline = chain.reload
|
|
67
|
+
branch_step = pipeline.steps.find_by(key: "format_check")
|
|
68
|
+
|
|
69
|
+
assert_equal "sd", branch_step.branch_result
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def test_branch_step_is_real_step_with_sentinel_job_class
|
|
73
|
+
chain = BranchTestPipeline.run(choice: "hd")
|
|
74
|
+
|
|
75
|
+
pipeline = chain.reload
|
|
76
|
+
branch_step = pipeline.steps.find_by(key: "format_check")
|
|
77
|
+
|
|
78
|
+
assert_predicate branch_step, :branch_step?
|
|
79
|
+
assert_equal GoodPipeline::BRANCH_JOB_CLASS, branch_step.job_class
|
|
80
|
+
assert_equal "pick_format", branch_step.decides
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def test_arm_steps_have_branch_metadata
|
|
84
|
+
chain = BranchTestPipeline.run(choice: "hd")
|
|
85
|
+
|
|
86
|
+
pipeline = chain.reload
|
|
87
|
+
arm_steps = pipeline.steps.select(&:branch_arm_step?)
|
|
88
|
+
|
|
89
|
+
assert_equal 3, arm_steps.size
|
|
90
|
+
arm_steps.each { |step| assert_equal "format_check", step.branch_key }
|
|
91
|
+
|
|
92
|
+
hd_steps = arm_steps.select { |step| step.branch_arm == "hd" }
|
|
93
|
+
sd_steps = arm_steps.select { |step| step.branch_arm == "sd" }
|
|
94
|
+
|
|
95
|
+
assert_equal 2, hd_steps.size
|
|
96
|
+
assert_equal 1, sd_steps.size
|
|
97
|
+
end
|
|
98
|
+
end
|