simple_flow 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.envrc +1 -0
- data/.github/workflows/deploy-github-pages.yml +52 -0
- data/.rubocop.yml +57 -0
- data/CHANGELOG.md +4 -0
- data/COMMITS.md +196 -0
- data/LICENSE +21 -0
- data/README.md +481 -0
- data/Rakefile +15 -0
- data/benchmarks/parallel_vs_sequential.rb +98 -0
- data/benchmarks/pipeline_overhead.rb +130 -0
- data/docs/api/middleware.md +468 -0
- data/docs/api/parallel-step.md +363 -0
- data/docs/api/pipeline.md +382 -0
- data/docs/api/result.md +375 -0
- data/docs/concurrent/best-practices.md +687 -0
- data/docs/concurrent/introduction.md +246 -0
- data/docs/concurrent/parallel-steps.md +418 -0
- data/docs/concurrent/performance.md +481 -0
- data/docs/core-concepts/flow-control.md +452 -0
- data/docs/core-concepts/middleware.md +389 -0
- data/docs/core-concepts/overview.md +219 -0
- data/docs/core-concepts/pipeline.md +315 -0
- data/docs/core-concepts/result.md +168 -0
- data/docs/core-concepts/steps.md +391 -0
- data/docs/development/benchmarking.md +443 -0
- data/docs/development/contributing.md +380 -0
- data/docs/development/dagwood-concepts.md +435 -0
- data/docs/development/testing.md +514 -0
- data/docs/getting-started/examples.md +197 -0
- data/docs/getting-started/installation.md +62 -0
- data/docs/getting-started/quick-start.md +218 -0
- data/docs/guides/choosing-concurrency-model.md +441 -0
- data/docs/guides/complex-workflows.md +440 -0
- data/docs/guides/data-fetching.md +478 -0
- data/docs/guides/error-handling.md +635 -0
- data/docs/guides/file-processing.md +505 -0
- data/docs/guides/validation-patterns.md +496 -0
- data/docs/index.md +169 -0
- data/examples/.gitignore +3 -0
- data/examples/01_basic_pipeline.rb +112 -0
- data/examples/02_error_handling.rb +178 -0
- data/examples/03_middleware.rb +186 -0
- data/examples/04_parallel_automatic.rb +221 -0
- data/examples/05_parallel_explicit.rb +279 -0
- data/examples/06_real_world_ecommerce.rb +288 -0
- data/examples/07_real_world_etl.rb +277 -0
- data/examples/08_graph_visualization.rb +246 -0
- data/examples/09_pipeline_visualization.rb +266 -0
- data/examples/10_concurrency_control.rb +235 -0
- data/examples/11_sequential_dependencies.rb +243 -0
- data/examples/12_none_constant.rb +161 -0
- data/examples/README.md +374 -0
- data/examples/regression_test/01_basic_pipeline.txt +38 -0
- data/examples/regression_test/02_error_handling.txt +92 -0
- data/examples/regression_test/03_middleware.txt +61 -0
- data/examples/regression_test/04_parallel_automatic.txt +86 -0
- data/examples/regression_test/05_parallel_explicit.txt +80 -0
- data/examples/regression_test/06_real_world_ecommerce.txt +53 -0
- data/examples/regression_test/07_real_world_etl.txt +58 -0
- data/examples/regression_test/08_graph_visualization.txt +429 -0
- data/examples/regression_test/09_pipeline_visualization.txt +305 -0
- data/examples/regression_test/10_concurrency_control.txt +96 -0
- data/examples/regression_test/11_sequential_dependencies.txt +86 -0
- data/examples/regression_test/12_none_constant.txt +64 -0
- data/examples/regression_test.rb +105 -0
- data/lib/simple_flow/dependency_graph.rb +120 -0
- data/lib/simple_flow/dependency_graph_visualizer.rb +326 -0
- data/lib/simple_flow/middleware.rb +36 -0
- data/lib/simple_flow/parallel_executor.rb +80 -0
- data/lib/simple_flow/pipeline.rb +405 -0
- data/lib/simple_flow/result.rb +88 -0
- data/lib/simple_flow/step_tracker.rb +58 -0
- data/lib/simple_flow/version.rb +5 -0
- data/lib/simple_flow.rb +41 -0
- data/mkdocs.yml +146 -0
- data/pipeline_graph.dot +51 -0
- data/pipeline_graph.html +60 -0
- data/pipeline_graph.mmd +19 -0
- metadata +127 -0
|
@@ -0,0 +1,452 @@
|
|
|
1
|
+
# Flow Control
|
|
2
|
+
|
|
3
|
+
Flow control in SimpleFlow allows you to manage the execution path of your pipeline based on conditions, errors, or business logic.
|
|
4
|
+
|
|
5
|
+
## Sequential Step Dependencies
|
|
6
|
+
|
|
7
|
+
**In sequential pipelines, each unnamed step automatically depends on the previous step's success.**
|
|
8
|
+
|
|
9
|
+
This means that steps execute in order, and the pipeline short-circuits (stops) as soon as any step halts:
|
|
10
|
+
|
|
11
|
+
```ruby
|
|
12
|
+
pipeline = SimpleFlow::Pipeline.new do
|
|
13
|
+
step ->(result) {
|
|
14
|
+
puts "Step 1: Running"
|
|
15
|
+
result.continue(result.value)
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
step ->(result) {
|
|
19
|
+
puts "Step 2: Halting"
|
|
20
|
+
result.halt("error occurred")
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
step ->(result) {
|
|
24
|
+
puts "Step 3: This never runs"
|
|
25
|
+
result.continue(result.value)
|
|
26
|
+
}
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
result = pipeline.call(SimpleFlow::Result.new(nil))
|
|
30
|
+
# Output:
|
|
31
|
+
# Step 1: Running
|
|
32
|
+
# Step 2: Halting
|
|
33
|
+
# (Step 3 is skipped)
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
**Key points:**
|
|
37
|
+
- No need to explicitly define dependencies for sequential workflows
|
|
38
|
+
- Each step receives the result from the previous step
|
|
39
|
+
- Halting a step prevents all subsequent steps from executing
|
|
40
|
+
- This is the default behavior for unnamed steps using `pipeline.call(result)`
|
|
41
|
+
|
|
42
|
+
## The Continue Flag
|
|
43
|
+
|
|
44
|
+
Every `Result` has a `continue?` method that determines whether the pipeline should proceed:
|
|
45
|
+
|
|
46
|
+
```ruby
|
|
47
|
+
result = SimpleFlow::Result.new(data)
|
|
48
|
+
result.continue? # => true (default)
|
|
49
|
+
|
|
50
|
+
result = result.halt
|
|
51
|
+
result.continue? # => false
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Halting Execution
|
|
55
|
+
|
|
56
|
+
### Basic Halt
|
|
57
|
+
|
|
58
|
+
Stop the pipeline while preserving the current value:
|
|
59
|
+
|
|
60
|
+
```ruby
|
|
61
|
+
step ->(result) do
|
|
62
|
+
if should_stop?(result.value)
|
|
63
|
+
return result.halt
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
result.continue(process(result.value))
|
|
67
|
+
end
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### Halt with New Value
|
|
71
|
+
|
|
72
|
+
Stop the pipeline with a different value (e.g., error response):
|
|
73
|
+
|
|
74
|
+
```ruby
|
|
75
|
+
step ->(result) do
|
|
76
|
+
unless valid?(result.value)
|
|
77
|
+
error_response = { error: 'Invalid data' }
|
|
78
|
+
return result.halt(error_response)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
result.continue(result.value)
|
|
82
|
+
end
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## Continue After Halt
|
|
86
|
+
|
|
87
|
+
Once halted, a result stays halted even if you try to continue:
|
|
88
|
+
|
|
89
|
+
```ruby
|
|
90
|
+
result = SimpleFlow::Result.new(data)
|
|
91
|
+
.halt
|
|
92
|
+
.continue('new value')
|
|
93
|
+
|
|
94
|
+
result.continue? # => false (still halted)
|
|
95
|
+
result.value # => 'new value' (value changed, but still halted)
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
## Conditional Execution
|
|
99
|
+
|
|
100
|
+
### Early Return Pattern
|
|
101
|
+
|
|
102
|
+
```ruby
|
|
103
|
+
step ->(result) do
|
|
104
|
+
# Skip processing if conditions not met
|
|
105
|
+
return result.continue(result.value) if skip_condition?(result)
|
|
106
|
+
|
|
107
|
+
# Process normally
|
|
108
|
+
processed = expensive_operation(result.value)
|
|
109
|
+
result.continue(processed)
|
|
110
|
+
end
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
### Guard Clauses
|
|
114
|
+
|
|
115
|
+
```ruby
|
|
116
|
+
step ->(result) do
|
|
117
|
+
data = result.value
|
|
118
|
+
|
|
119
|
+
# Multiple guard clauses
|
|
120
|
+
return result.with_error(:validation, 'ID required').halt unless data[:id]
|
|
121
|
+
return result.with_error(:validation, 'Email required').halt unless data[:email]
|
|
122
|
+
return result.with_error(:authorization, 'Unauthorized').halt unless authorized?(data)
|
|
123
|
+
|
|
124
|
+
# Main logic
|
|
125
|
+
result.continue(process(data))
|
|
126
|
+
end
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
### Branching Logic
|
|
130
|
+
|
|
131
|
+
```ruby
|
|
132
|
+
step ->(result) do
|
|
133
|
+
user_type = result.value[:type]
|
|
134
|
+
|
|
135
|
+
case user_type
|
|
136
|
+
when 'premium'
|
|
137
|
+
result.continue(process_premium(result.value))
|
|
138
|
+
when 'standard'
|
|
139
|
+
result.continue(process_standard(result.value))
|
|
140
|
+
when 'trial'
|
|
141
|
+
result.continue(process_trial(result.value))
|
|
142
|
+
else
|
|
143
|
+
result.with_error(:validation, "Unknown type: #{user_type}").halt
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
## Error-Based Flow Control
|
|
149
|
+
|
|
150
|
+
### Accumulate Errors, Continue Processing
|
|
151
|
+
|
|
152
|
+
```ruby
|
|
153
|
+
step ->(result) do
|
|
154
|
+
data = result.value
|
|
155
|
+
result_with_errors = result
|
|
156
|
+
|
|
157
|
+
# Collect all validation errors
|
|
158
|
+
if data[:name].blank?
|
|
159
|
+
result_with_errors = result_with_errors.with_error(:validation, 'Name required')
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
if data[:email].blank?
|
|
163
|
+
result_with_errors = result_with_errors.with_error(:validation, 'Email required')
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
if data[:age] && data[:age] < 18
|
|
167
|
+
result_with_errors = result_with_errors.with_error(:validation, 'Must be 18+')
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# Continue with errors tracked
|
|
171
|
+
result_with_errors.continue(data)
|
|
172
|
+
end
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
### Halt on Critical Errors
|
|
176
|
+
|
|
177
|
+
```ruby
|
|
178
|
+
step ->(result) do
|
|
179
|
+
data = result.value
|
|
180
|
+
result_with_errors = result
|
|
181
|
+
|
|
182
|
+
# Collect warnings (non-critical)
|
|
183
|
+
if data[:phone].blank?
|
|
184
|
+
result_with_errors = result_with_errors.with_error(:warning, 'Phone number recommended')
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# Halt on critical errors
|
|
188
|
+
if data[:credit_card].blank?
|
|
189
|
+
return result_with_errors
|
|
190
|
+
.with_error(:critical, 'Payment method required')
|
|
191
|
+
.halt
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
result_with_errors.continue(data)
|
|
195
|
+
end
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
### Check Accumulated Errors
|
|
199
|
+
|
|
200
|
+
```ruby
|
|
201
|
+
step ->(result) do
|
|
202
|
+
# Check if previous steps added errors
|
|
203
|
+
if result.errors.key?(:validation)
|
|
204
|
+
return result.halt # Stop if validation errors exist
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
result.continue(process(result.value))
|
|
208
|
+
end
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
## Context-Based Flow Control
|
|
212
|
+
|
|
213
|
+
### Skip Steps Based on Context
|
|
214
|
+
|
|
215
|
+
```ruby
|
|
216
|
+
step ->(result) do
|
|
217
|
+
# Skip if already processed
|
|
218
|
+
if result.context[:processed]
|
|
219
|
+
return result.continue(result.value)
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
processed = process_data(result.value)
|
|
223
|
+
result
|
|
224
|
+
.continue(processed)
|
|
225
|
+
.with_context(:processed, true)
|
|
226
|
+
end
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
### Feature Flags
|
|
230
|
+
|
|
231
|
+
```ruby
|
|
232
|
+
step ->(result) do
|
|
233
|
+
# Skip if feature disabled
|
|
234
|
+
unless result.context[:feature_enabled]
|
|
235
|
+
return result.continue(result.value)
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
new_feature_processing(result.value)
|
|
239
|
+
result.continue(processed)
|
|
240
|
+
end
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
## Retry Logic
|
|
244
|
+
|
|
245
|
+
### Simple Retry
|
|
246
|
+
|
|
247
|
+
```ruby
|
|
248
|
+
step ->(result) do
|
|
249
|
+
max_retries = 3
|
|
250
|
+
attempts = 0
|
|
251
|
+
|
|
252
|
+
begin
|
|
253
|
+
data = unreliable_api_call(result.value)
|
|
254
|
+
result.continue(data)
|
|
255
|
+
rescue StandardError => e
|
|
256
|
+
attempts += 1
|
|
257
|
+
retry if attempts < max_retries
|
|
258
|
+
|
|
259
|
+
result
|
|
260
|
+
.with_error(:api, "Failed after #{attempts} attempts: #{e.message}")
|
|
261
|
+
.halt
|
|
262
|
+
end
|
|
263
|
+
end
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
### Exponential Backoff
|
|
267
|
+
|
|
268
|
+
```ruby
|
|
269
|
+
step ->(result) do
|
|
270
|
+
max_retries = 5
|
|
271
|
+
base_delay = 1.0
|
|
272
|
+
attempts = 0
|
|
273
|
+
|
|
274
|
+
begin
|
|
275
|
+
data = fetch_external_data(result.value)
|
|
276
|
+
result.continue(data)
|
|
277
|
+
rescue StandardError => e
|
|
278
|
+
attempts += 1
|
|
279
|
+
|
|
280
|
+
if attempts < max_retries
|
|
281
|
+
delay = base_delay * (2 ** (attempts - 1))
|
|
282
|
+
sleep(delay)
|
|
283
|
+
retry
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
result
|
|
287
|
+
.with_error(:external, "Max retries exceeded: #{e.message}")
|
|
288
|
+
.with_context(:retry_attempts, attempts)
|
|
289
|
+
.halt
|
|
290
|
+
end
|
|
291
|
+
end
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
## Circuit Breaker Pattern
|
|
295
|
+
|
|
296
|
+
```ruby
|
|
297
|
+
class CircuitBreaker
|
|
298
|
+
def initialize
|
|
299
|
+
@failure_count = 0
|
|
300
|
+
@last_failure_time = nil
|
|
301
|
+
@threshold = 5
|
|
302
|
+
@timeout = 60
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
def call(result)
|
|
306
|
+
# Check if circuit is open
|
|
307
|
+
if circuit_open?
|
|
308
|
+
return result
|
|
309
|
+
.with_error(:circuit_breaker, 'Circuit breaker open')
|
|
310
|
+
.halt
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
# Try operation
|
|
314
|
+
begin
|
|
315
|
+
data = risky_operation(result.value)
|
|
316
|
+
reset_circuit
|
|
317
|
+
result.continue(data)
|
|
318
|
+
rescue StandardError => e
|
|
319
|
+
record_failure
|
|
320
|
+
result.with_error(:operation, e.message).halt
|
|
321
|
+
end
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
private
|
|
325
|
+
|
|
326
|
+
def circuit_open?
|
|
327
|
+
@failure_count >= @threshold &&
|
|
328
|
+
@last_failure_time &&
|
|
329
|
+
(Time.now - @last_failure_time) < @timeout
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
def record_failure
|
|
333
|
+
@failure_count += 1
|
|
334
|
+
@last_failure_time = Time.now
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
def reset_circuit
|
|
338
|
+
@failure_count = 0
|
|
339
|
+
@last_failure_time = nil
|
|
340
|
+
end
|
|
341
|
+
end
|
|
342
|
+
```
|
|
343
|
+
|
|
344
|
+
## Conditional Pipeline Construction
|
|
345
|
+
|
|
346
|
+
Build pipelines dynamically based on conditions:
|
|
347
|
+
|
|
348
|
+
```ruby
|
|
349
|
+
def build_pipeline(user_type)
|
|
350
|
+
SimpleFlow::Pipeline.new do
|
|
351
|
+
# Always validate
|
|
352
|
+
step method(:validate_user)
|
|
353
|
+
|
|
354
|
+
# Conditional steps based on user type
|
|
355
|
+
if user_type == :premium
|
|
356
|
+
step method(:apply_premium_discount)
|
|
357
|
+
step method(:check_premium_limits)
|
|
358
|
+
end
|
|
359
|
+
|
|
360
|
+
if user_type == :enterprise
|
|
361
|
+
step method(:check_bulk_pricing)
|
|
362
|
+
step method(:assign_account_manager)
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
# Always process
|
|
366
|
+
step method(:process_order)
|
|
367
|
+
end
|
|
368
|
+
end
|
|
369
|
+
```
|
|
370
|
+
|
|
371
|
+
## Short-Circuit Entire Pipeline
|
|
372
|
+
|
|
373
|
+
```ruby
|
|
374
|
+
pipeline = SimpleFlow::Pipeline.new do
|
|
375
|
+
# Pre-flight check - halts entire pipeline if fails
|
|
376
|
+
step ->(result) do
|
|
377
|
+
unless system_healthy?
|
|
378
|
+
return result
|
|
379
|
+
.with_error(:system, 'System maintenance in progress')
|
|
380
|
+
.halt
|
|
381
|
+
end
|
|
382
|
+
result.continue(result.value)
|
|
383
|
+
end
|
|
384
|
+
|
|
385
|
+
# These only run if pre-flight passes
|
|
386
|
+
step method(:process_data)
|
|
387
|
+
step method(:validate_results)
|
|
388
|
+
step method(:save_to_database)
|
|
389
|
+
end
|
|
390
|
+
```
|
|
391
|
+
|
|
392
|
+
## Best Practices
|
|
393
|
+
|
|
394
|
+
1. **Fail Fast**: Use `halt` as soon as you know processing cannot succeed
|
|
395
|
+
2. **Preserve Context**: Keep error messages and context for debugging
|
|
396
|
+
3. **Distinguish Error Severity**: Use different error categories (`:validation`, `:critical`, `:warning`)
|
|
397
|
+
4. **Use Early Returns**: Make guard clauses clear with early returns
|
|
398
|
+
5. **Document Flow Logic**: Comment complex conditional logic
|
|
399
|
+
6. **Test Both Paths**: Test both success and failure paths
|
|
400
|
+
7. **Avoid Deep Nesting**: Use early returns instead of nested conditionals
|
|
401
|
+
|
|
402
|
+
## Common Patterns
|
|
403
|
+
|
|
404
|
+
### Validation Pipeline
|
|
405
|
+
|
|
406
|
+
```ruby
|
|
407
|
+
pipeline = SimpleFlow::Pipeline.new do
|
|
408
|
+
step ->(result) do
|
|
409
|
+
# Collect all errors, but don't halt yet
|
|
410
|
+
result_with_errors = validate_all_fields(result)
|
|
411
|
+
|
|
412
|
+
# Halt only if errors exist
|
|
413
|
+
if result_with_errors.errors.any?
|
|
414
|
+
return result_with_errors.halt
|
|
415
|
+
end
|
|
416
|
+
|
|
417
|
+
result_with_errors.continue(result.value)
|
|
418
|
+
end
|
|
419
|
+
|
|
420
|
+
# This only runs if validation passed
|
|
421
|
+
step method(:process_valid_data)
|
|
422
|
+
end
|
|
423
|
+
```
|
|
424
|
+
|
|
425
|
+
### Multi-Stage Processing
|
|
426
|
+
|
|
427
|
+
```ruby
|
|
428
|
+
pipeline = SimpleFlow::Pipeline.new do
|
|
429
|
+
# Stage 1: Preparation (must succeed)
|
|
430
|
+
step method(:fetch_data)
|
|
431
|
+
step method(:validate_data)
|
|
432
|
+
|
|
433
|
+
# Stage 2: Processing (optional based on flags)
|
|
434
|
+
step ->(result) do
|
|
435
|
+
if result.context[:skip_processing]
|
|
436
|
+
return result.continue(result.value)
|
|
437
|
+
end
|
|
438
|
+
result.continue(process_data(result.value))
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
# Stage 3: Finalization (always runs if we got here)
|
|
442
|
+
step method(:save_results)
|
|
443
|
+
step method(:send_notifications)
|
|
444
|
+
end
|
|
445
|
+
```
|
|
446
|
+
|
|
447
|
+
## Next Steps
|
|
448
|
+
|
|
449
|
+
- [Result](result.md) - Understanding the Result object
|
|
450
|
+
- [Steps](steps.md) - Implementing step logic
|
|
451
|
+
- [Error Handling Guide](../guides/error-handling.md) - Comprehensive error handling strategies
|
|
452
|
+
- [Complex Workflows Guide](../guides/complex-workflows.md) - Real-world flow control examples
|