simple_flow 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +7 -0
  2. data/.envrc +1 -0
  3. data/.github/workflows/deploy-github-pages.yml +52 -0
  4. data/.rubocop.yml +57 -0
  5. data/CHANGELOG.md +4 -0
  6. data/COMMITS.md +196 -0
  7. data/LICENSE +21 -0
  8. data/README.md +481 -0
  9. data/Rakefile +15 -0
  10. data/benchmarks/parallel_vs_sequential.rb +98 -0
  11. data/benchmarks/pipeline_overhead.rb +130 -0
  12. data/docs/api/middleware.md +468 -0
  13. data/docs/api/parallel-step.md +363 -0
  14. data/docs/api/pipeline.md +382 -0
  15. data/docs/api/result.md +375 -0
  16. data/docs/concurrent/best-practices.md +687 -0
  17. data/docs/concurrent/introduction.md +246 -0
  18. data/docs/concurrent/parallel-steps.md +418 -0
  19. data/docs/concurrent/performance.md +481 -0
  20. data/docs/core-concepts/flow-control.md +452 -0
  21. data/docs/core-concepts/middleware.md +389 -0
  22. data/docs/core-concepts/overview.md +219 -0
  23. data/docs/core-concepts/pipeline.md +315 -0
  24. data/docs/core-concepts/result.md +168 -0
  25. data/docs/core-concepts/steps.md +391 -0
  26. data/docs/development/benchmarking.md +443 -0
  27. data/docs/development/contributing.md +380 -0
  28. data/docs/development/dagwood-concepts.md +435 -0
  29. data/docs/development/testing.md +514 -0
  30. data/docs/getting-started/examples.md +197 -0
  31. data/docs/getting-started/installation.md +62 -0
  32. data/docs/getting-started/quick-start.md +218 -0
  33. data/docs/guides/choosing-concurrency-model.md +441 -0
  34. data/docs/guides/complex-workflows.md +440 -0
  35. data/docs/guides/data-fetching.md +478 -0
  36. data/docs/guides/error-handling.md +635 -0
  37. data/docs/guides/file-processing.md +505 -0
  38. data/docs/guides/validation-patterns.md +496 -0
  39. data/docs/index.md +169 -0
  40. data/examples/.gitignore +3 -0
  41. data/examples/01_basic_pipeline.rb +112 -0
  42. data/examples/02_error_handling.rb +178 -0
  43. data/examples/03_middleware.rb +186 -0
  44. data/examples/04_parallel_automatic.rb +221 -0
  45. data/examples/05_parallel_explicit.rb +279 -0
  46. data/examples/06_real_world_ecommerce.rb +288 -0
  47. data/examples/07_real_world_etl.rb +277 -0
  48. data/examples/08_graph_visualization.rb +246 -0
  49. data/examples/09_pipeline_visualization.rb +266 -0
  50. data/examples/10_concurrency_control.rb +235 -0
  51. data/examples/11_sequential_dependencies.rb +243 -0
  52. data/examples/12_none_constant.rb +161 -0
  53. data/examples/README.md +374 -0
  54. data/examples/regression_test/01_basic_pipeline.txt +38 -0
  55. data/examples/regression_test/02_error_handling.txt +92 -0
  56. data/examples/regression_test/03_middleware.txt +61 -0
  57. data/examples/regression_test/04_parallel_automatic.txt +86 -0
  58. data/examples/regression_test/05_parallel_explicit.txt +80 -0
  59. data/examples/regression_test/06_real_world_ecommerce.txt +53 -0
  60. data/examples/regression_test/07_real_world_etl.txt +58 -0
  61. data/examples/regression_test/08_graph_visualization.txt +429 -0
  62. data/examples/regression_test/09_pipeline_visualization.txt +305 -0
  63. data/examples/regression_test/10_concurrency_control.txt +96 -0
  64. data/examples/regression_test/11_sequential_dependencies.txt +86 -0
  65. data/examples/regression_test/12_none_constant.txt +64 -0
  66. data/examples/regression_test.rb +105 -0
  67. data/lib/simple_flow/dependency_graph.rb +120 -0
  68. data/lib/simple_flow/dependency_graph_visualizer.rb +326 -0
  69. data/lib/simple_flow/middleware.rb +36 -0
  70. data/lib/simple_flow/parallel_executor.rb +80 -0
  71. data/lib/simple_flow/pipeline.rb +405 -0
  72. data/lib/simple_flow/result.rb +88 -0
  73. data/lib/simple_flow/step_tracker.rb +58 -0
  74. data/lib/simple_flow/version.rb +5 -0
  75. data/lib/simple_flow.rb +41 -0
  76. data/mkdocs.yml +146 -0
  77. data/pipeline_graph.dot +51 -0
  78. data/pipeline_graph.html +60 -0
  79. data/pipeline_graph.mmd +19 -0
  80. metadata +127 -0
@@ -0,0 +1,130 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require 'bundler/setup'
5
+ require 'benchmark/ips'
6
+ require_relative '../lib/simple_flow'
7
+
8
+ puts '=' * 80
9
+ puts 'SimpleFlow Performance Benchmarks: Pipeline Overhead'
10
+ puts '=' * 80
11
+ puts
12
+
13
+ # Simple operation for measuring overhead
14
+ simple_op = ->(x) { x + 1 }
15
+
16
+ # Baseline: raw operations
17
+ def raw_operations(value)
18
+ value += 1
19
+ value += 1
20
+ value += 1
21
+ value += 1
22
+ value
23
+ end
24
+
25
+ # Pipeline operations
26
+ pipeline = SimpleFlow::Pipeline.new do
27
+ step ->(result) { result.continue(result.value + 1) }
28
+ step ->(result) { result.continue(result.value + 1) }
29
+ step ->(result) { result.continue(result.value + 1) }
30
+ step ->(result) { result.continue(result.value + 1) }
31
+ end
32
+
33
+ puts "Comparing raw operations vs Pipeline"
34
+ puts
35
+
36
+ Benchmark.ips do |x|
37
+ x.config(time: 5, warmup: 2)
38
+
39
+ x.report('raw operations') do
40
+ raw_operations(0)
41
+ end
42
+
43
+ x.report('pipeline') do
44
+ pipeline.call(SimpleFlow::Result.new(0))
45
+ end
46
+
47
+ x.compare!
48
+ end
49
+
50
+ puts "\n" + '=' * 80
51
+ puts
52
+
53
+ # Middleware overhead
54
+ puts 'Benchmark: Middleware Overhead'
55
+ puts '=' * 80
56
+ puts
57
+
58
+ no_middleware = SimpleFlow::Pipeline.new do
59
+ step ->(result) { result.continue(result.value + 1) }
60
+ step ->(result) { result.continue(result.value + 1) }
61
+ step ->(result) { result.continue(result.value + 1) }
62
+ end
63
+
64
+ with_middleware = SimpleFlow::Pipeline.new do
65
+ use_middleware SimpleFlow::MiddleWare::Instrumentation, api_key: 'test'
66
+
67
+ step ->(result) { result.continue(result.value + 1) }
68
+ step ->(result) { result.continue(result.value + 1) }
69
+ step ->(result) { result.continue(result.value + 1) }
70
+ end
71
+
72
+ Benchmark.ips do |x|
73
+ x.config(time: 5, warmup: 2)
74
+
75
+ x.report('no middleware') do
76
+ no_middleware.call(SimpleFlow::Result.new(0))
77
+ end
78
+
79
+ x.report('with instrumentation') do |times|
80
+ # Suppress output
81
+ original_stdout = $stdout
82
+ $stdout = File.open(File::NULL, 'w')
83
+ times.times do
84
+ with_middleware.call(SimpleFlow::Result.new(0))
85
+ end
86
+ $stdout = original_stdout
87
+ end
88
+
89
+ x.compare!
90
+ end
91
+
92
+ puts "\n" + '=' * 80
93
+ puts
94
+
95
+ # Result creation overhead
96
+ puts 'Benchmark: Result Creation and Immutability'
97
+ puts '=' * 80
98
+ puts
99
+
100
+ Benchmark.ips do |x|
101
+ x.config(time: 5, warmup: 2)
102
+
103
+ x.report('new result') do
104
+ SimpleFlow::Result.new(42)
105
+ end
106
+
107
+ x.report('with_context') do
108
+ result = SimpleFlow::Result.new(42)
109
+ result.with_context(:key, 'value')
110
+ end
111
+
112
+ x.report('with_error') do
113
+ result = SimpleFlow::Result.new(42)
114
+ result.with_error(:error, 'message')
115
+ end
116
+
117
+ x.report('continue') do
118
+ result = SimpleFlow::Result.new(42)
119
+ result.continue(43)
120
+ end
121
+
122
+ x.report('halt') do
123
+ result = SimpleFlow::Result.new(42)
124
+ result.halt
125
+ end
126
+
127
+ x.compare!
128
+ end
129
+
130
+ puts "\n" + '=' * 80
@@ -0,0 +1,468 @@
1
+ # Middleware API Reference
2
+
3
+ Middleware in SimpleFlow wraps steps with cross-cutting functionality using the decorator pattern. This document covers built-in middleware and how to create custom middleware.
4
+
5
+ ## Built-in Middleware
6
+
7
+ ### Class: `SimpleFlow::MiddleWare::Logging`
8
+
9
+ **Location**: `/Users/dewayne/sandbox/git_repos/madbomber/simple_flow/lib/simple_flow/middleware.rb`
10
+
11
+ Logs before and after step execution.
12
+
13
+ #### Constructor
14
+
15
+ ```ruby
16
+ def initialize(callable, logger = nil)
17
+ ```
18
+
19
+ **Parameters:**
20
+ - `callable` (Proc/Object) - The step to wrap
21
+ - `logger` (Logger, optional) - Custom logger instance (default: `Logger.new($stdout)`)
22
+
23
+ #### Usage
24
+
25
+ ```ruby
26
+ pipeline = SimpleFlow::Pipeline.new do
27
+ use_middleware SimpleFlow::MiddleWare::Logging
28
+
29
+ step ->(result) { result.continue(process(result.value)) }
30
+ end
31
+ ```
32
+
33
+ **With Custom Logger:**
34
+ ```ruby
35
+ require 'logger'
36
+
37
+ custom_logger = Logger.new('pipeline.log')
38
+ custom_logger.level = Logger::DEBUG
39
+
40
+ pipeline = SimpleFlow::Pipeline.new do
41
+ use_middleware SimpleFlow::MiddleWare::Logging, logger: custom_logger
42
+
43
+ step ->(result) { result.continue(result.value) }
44
+ end
45
+ ```
46
+
47
+ **Output:**
48
+ ```
49
+ I, [2025-11-15T12:00:00.123456 #12345] INFO -- : Before call
50
+ I, [2025-11-15T12:00:00.456789 #12345] INFO -- : After call
51
+ ```
52
+
53
+ ### Class: `SimpleFlow::MiddleWare::Instrumentation`
54
+
55
+ **Location**: `/Users/dewayne/sandbox/git_repos/madbomber/simple_flow/lib/simple_flow/middleware.rb`
56
+
57
+ Measures step execution duration.
58
+
59
+ #### Constructor
60
+
61
+ ```ruby
62
+ def initialize(callable, api_key: nil)
63
+ ```
64
+
65
+ **Parameters:**
66
+ - `callable` (Proc/Object) - The step to wrap
67
+ - `api_key` (String, optional) - API key for external instrumentation service
68
+
69
+ #### Usage
70
+
71
+ ```ruby
72
+ pipeline = SimpleFlow::Pipeline.new do
73
+ use_middleware SimpleFlow::MiddleWare::Instrumentation, api_key: 'demo-key-123'
74
+
75
+ step ->(result) {
76
+ sleep 0.1
77
+ result.continue(result.value)
78
+ }
79
+ end
80
+ ```
81
+
82
+ **Output:**
83
+ ```
84
+ Instrumentation: demo-key-123 took 0.10012345s
85
+ ```
86
+
87
+ ## Creating Custom Middleware
88
+
89
+ ### Basic Pattern
90
+
91
+ Custom middleware must implement a `call` method that:
92
+ 1. Receives a Result object
93
+ 2. Calls the wrapped callable
94
+ 3. Returns a Result object
95
+
96
+ ```ruby
97
+ class MyMiddleware
98
+ def initialize(callable, **options)
99
+ @callable = callable
100
+ @options = options
101
+ end
102
+
103
+ def call(result)
104
+ # Before step execution
105
+ puts "Before: #{result.value}"
106
+
107
+ # Execute the wrapped step
108
+ output = @callable.call(result)
109
+
110
+ # After step execution
111
+ puts "After: #{output.value}"
112
+
113
+ output
114
+ end
115
+ end
116
+
117
+ # Usage
118
+ pipeline = SimpleFlow::Pipeline.new do
119
+ use_middleware MyMiddleware, option: "value"
120
+
121
+ step ->(result) { result.continue(result.value) }
122
+ end
123
+ ```
124
+
125
+ ### Middleware Examples
126
+
127
+ #### Timing Middleware
128
+
129
+ ```ruby
130
+ class TimingMiddleware
131
+ def initialize(callable, step_name: nil)
132
+ @callable = callable
133
+ @step_name = step_name || "unknown_step"
134
+ end
135
+
136
+ def call(result)
137
+ start_time = Time.now
138
+ output = @callable.call(result)
139
+ duration = Time.now - start_time
140
+
141
+ output.with_context(
142
+ "#{@step_name}_duration".to_sym,
143
+ duration
144
+ )
145
+ end
146
+ end
147
+
148
+ # Usage
149
+ pipeline = SimpleFlow::Pipeline.new do
150
+ use_middleware TimingMiddleware, step_name: "data_processing"
151
+
152
+ step ->(result) {
153
+ process_data(result.value)
154
+ result.continue(result.value)
155
+ }
156
+ end
157
+
158
+ result = pipeline.call(initial_data)
159
+ puts "Execution time: #{result.context[:data_processing_duration]}s"
160
+ ```
161
+
162
+ #### Retry Middleware
163
+
164
+ ```ruby
165
+ class RetryMiddleware
166
+ def initialize(callable, max_retries: 3, retry_on: [StandardError])
167
+ @callable = callable
168
+ @max_retries = max_retries
169
+ @retry_on = Array(retry_on)
170
+ end
171
+
172
+ def call(result)
173
+ attempts = 0
174
+
175
+ begin
176
+ attempts += 1
177
+ @callable.call(result)
178
+ rescue *@retry_on => e
179
+ if attempts < @max_retries
180
+ sleep(attempts ** 2) # Exponential backoff
181
+ retry
182
+ else
183
+ result.halt.with_error(
184
+ :retry_exhausted,
185
+ "Failed after #{@max_retries} attempts: #{e.message}"
186
+ )
187
+ end
188
+ end
189
+ end
190
+ end
191
+
192
+ # Usage
193
+ pipeline = SimpleFlow::Pipeline.new do
194
+ use_middleware RetryMiddleware, max_retries: 3, retry_on: [Net::HTTPError]
195
+
196
+ step ->(result) {
197
+ data = fetch_from_api(result.value) # May fail temporarily
198
+ result.continue(data)
199
+ }
200
+ end
201
+ ```
202
+
203
+ #### Authentication Middleware
204
+
205
+ ```ruby
206
+ class AuthMiddleware
207
+ def initialize(callable, required_role:)
208
+ @callable = callable
209
+ @required_role = required_role
210
+ end
211
+
212
+ def call(result)
213
+ user_role = result.context[:user_role]
214
+
215
+ unless user_role == @required_role
216
+ return result.halt.with_error(
217
+ :auth,
218
+ "Unauthorized: requires #{@required_role} role"
219
+ )
220
+ end
221
+
222
+ @callable.call(result)
223
+ end
224
+ end
225
+
226
+ # Usage
227
+ pipeline = SimpleFlow::Pipeline.new do
228
+ # Set user role in first step
229
+ step ->(result) {
230
+ result.with_context(:user_role, :admin).continue(result.value)
231
+ }
232
+
233
+ # Protect subsequent steps
234
+ use_middleware AuthMiddleware, required_role: :admin
235
+
236
+ step ->(result) {
237
+ # This only executes if user_role == :admin
238
+ result.continue("Sensitive operation")
239
+ }
240
+ end
241
+ ```
242
+
243
+ #### Caching Middleware
244
+
245
+ ```ruby
246
+ class CachingMiddleware
247
+ def initialize(callable, cache_key_proc:, ttl: 3600)
248
+ @callable = callable
249
+ @cache_key_proc = cache_key_proc
250
+ @ttl = ttl
251
+ end
252
+
253
+ def call(result)
254
+ cache_key = @cache_key_proc.call(result)
255
+
256
+ # Check cache
257
+ if cached = REDIS.get(cache_key)
258
+ return result.with_context(:cache_hit, true).continue(JSON.parse(cached))
259
+ end
260
+
261
+ # Execute step
262
+ output = @callable.call(result)
263
+
264
+ # Cache result if successful
265
+ if output.continue?
266
+ REDIS.setex(cache_key, @ttl, output.value.to_json)
267
+ end
268
+
269
+ output.with_context(:cache_hit, false)
270
+ end
271
+ end
272
+
273
+ # Usage
274
+ pipeline = SimpleFlow::Pipeline.new do
275
+ use_middleware CachingMiddleware,
276
+ cache_key_proc: ->(result) { "user_#{result.value}" },
277
+ ttl: 1800
278
+
279
+ step ->(result) {
280
+ user = User.find(result.value)
281
+ result.continue(user)
282
+ }
283
+ end
284
+ ```
285
+
286
+ #### Error Tracking Middleware
287
+
288
+ ```ruby
289
+ class ErrorTrackingMiddleware
290
+ def initialize(callable, error_tracker:)
291
+ @callable = callable
292
+ @error_tracker = error_tracker
293
+ end
294
+
295
+ def call(result)
296
+ output = @callable.call(result)
297
+
298
+ # Report errors to tracking service
299
+ if !output.continue? && output.errors.any?
300
+ @error_tracker.report(
301
+ errors: output.errors,
302
+ context: output.context,
303
+ value: output.value
304
+ )
305
+ end
306
+
307
+ output
308
+ end
309
+ end
310
+
311
+ # Usage
312
+ pipeline = SimpleFlow::Pipeline.new do
313
+ use_middleware ErrorTrackingMiddleware, error_tracker: Sentry
314
+
315
+ step ->(result) {
316
+ # Errors here will be reported to Sentry
317
+ result.halt.with_error(:processing, "Something went wrong")
318
+ }
319
+ end
320
+ ```
321
+
322
+ ## Middleware Stacking
323
+
324
+ Middleware is applied in reverse order (last declared middleware wraps first):
325
+
326
+ ```ruby
327
+ pipeline = SimpleFlow::Pipeline.new do
328
+ use_middleware OuterMiddleware # Applied third (outermost)
329
+ use_middleware MiddleMiddleware # Applied second
330
+ use_middleware InnerMiddleware # Applied first (innermost)
331
+
332
+ step ->(result) { result.continue(result.value) }
333
+ end
334
+
335
+ # Execution order:
336
+ # 1. OuterMiddleware before
337
+ # 2. MiddleMiddleware before
338
+ # 3. InnerMiddleware before
339
+ # 4. Step execution
340
+ # 5. InnerMiddleware after
341
+ # 6. MiddleMiddleware after
342
+ # 7. OuterMiddleware after
343
+ ```
344
+
345
+ **Example:**
346
+ ```ruby
347
+ class LoggingMiddleware
348
+ def initialize(callable, name:)
349
+ @callable = callable
350
+ @name = name
351
+ end
352
+
353
+ def call(result)
354
+ puts "#{@name}: before"
355
+ output = @callable.call(result)
356
+ puts "#{@name}: after"
357
+ output
358
+ end
359
+ end
360
+
361
+ pipeline = SimpleFlow::Pipeline.new do
362
+ use_middleware LoggingMiddleware, name: "Outer"
363
+ use_middleware LoggingMiddleware, name: "Middle"
364
+ use_middleware LoggingMiddleware, name: "Inner"
365
+
366
+ step ->(result) {
367
+ puts "Step execution"
368
+ result.continue(result.value)
369
+ }
370
+ end
371
+
372
+ pipeline.call(SimpleFlow::Result.new(nil))
373
+
374
+ # Output:
375
+ # Outer: before
376
+ # Middle: before
377
+ # Inner: before
378
+ # Step execution
379
+ # Inner: after
380
+ # Middle: after
381
+ # Outer: after
382
+ ```
383
+
384
+ ## Best Practices
385
+
386
+ ### 1. Keep Middleware Focused
387
+
388
+ Each middleware should have a single responsibility:
389
+
390
+ ```ruby
391
+ # GOOD: Focused middleware
392
+ class TimingMiddleware
393
+ def call(result)
394
+ start = Time.now
395
+ output = @callable.call(result)
396
+ output.with_context(:duration, Time.now - start)
397
+ end
398
+ end
399
+
400
+ # BAD: Too many responsibilities
401
+ class KitchenSinkMiddleware
402
+ def call(result)
403
+ # Logging, timing, caching, retrying, auth... too much!
404
+ end
405
+ end
406
+ ```
407
+
408
+ ### 2. Preserve Result Immutability
409
+
410
+ Always return new Result objects:
411
+
412
+ ```ruby
413
+ # GOOD: Returns new Result
414
+ def call(result)
415
+ output = @callable.call(result)
416
+ output.with_context(:middleware_applied, true)
417
+ end
418
+
419
+ # BAD: Attempts to modify Result
420
+ def call(result)
421
+ output = @callable.call(result)
422
+ output.context[:middleware_applied] = true # Won't work!
423
+ output
424
+ end
425
+ ```
426
+
427
+ ### 3. Handle Errors Gracefully
428
+
429
+ Ensure middleware doesn't break the pipeline:
430
+
431
+ ```ruby
432
+ class SafeMiddleware
433
+ def call(result)
434
+ begin
435
+ @callable.call(result)
436
+ rescue StandardError => e
437
+ result.halt.with_error(:middleware_error, "Middleware failed: #{e.message}")
438
+ end
439
+ end
440
+ end
441
+ ```
442
+
443
+ ### 4. Make Middleware Configurable
444
+
445
+ Use options for flexibility:
446
+
447
+ ```ruby
448
+ class ConfigurableMiddleware
449
+ def initialize(callable, enabled: true, **options)
450
+ @callable = callable
451
+ @enabled = enabled
452
+ @options = options
453
+ end
454
+
455
+ def call(result)
456
+ return @callable.call(result) unless @enabled
457
+
458
+ # Middleware logic here
459
+ @callable.call(result)
460
+ end
461
+ end
462
+ ```
463
+
464
+ ## Related Documentation
465
+
466
+ - [Pipeline API](pipeline.md) - How pipelines use middleware
467
+ - [Complex Workflows](../guides/complex-workflows.md) - Using middleware in workflows
468
+ - [Error Handling](../guides/error-handling.md) - Error handling patterns