ruby_reactor 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +10 -2
- data/README.md +177 -3
- data/Rakefile +25 -0
- data/documentation/data_pipelines.md +90 -84
- data/documentation/images/failed_order_processing.png +0 -0
- data/documentation/images/payment_workflow.png +0 -0
- data/documentation/interrupts.md +161 -0
- data/gui/.gitignore +24 -0
- data/gui/README.md +73 -0
- data/gui/eslint.config.js +23 -0
- data/gui/index.html +13 -0
- data/gui/package-lock.json +5925 -0
- data/gui/package.json +46 -0
- data/gui/postcss.config.js +6 -0
- data/gui/public/vite.svg +1 -0
- data/gui/src/App.css +42 -0
- data/gui/src/App.tsx +51 -0
- data/gui/src/assets/react.svg +1 -0
- data/gui/src/components/DagVisualizer.tsx +424 -0
- data/gui/src/components/Dashboard.tsx +163 -0
- data/gui/src/components/ErrorBoundary.tsx +47 -0
- data/gui/src/components/ReactorDetail.tsx +135 -0
- data/gui/src/components/StepInspector.tsx +492 -0
- data/gui/src/components/__tests__/DagVisualizer.test.tsx +140 -0
- data/gui/src/components/__tests__/ReactorDetail.test.tsx +111 -0
- data/gui/src/components/__tests__/StepInspector.test.tsx +408 -0
- data/gui/src/globals.d.ts +7 -0
- data/gui/src/index.css +14 -0
- data/gui/src/lib/utils.ts +13 -0
- data/gui/src/main.tsx +14 -0
- data/gui/src/test/setup.ts +11 -0
- data/gui/tailwind.config.js +11 -0
- data/gui/tsconfig.app.json +28 -0
- data/gui/tsconfig.json +7 -0
- data/gui/tsconfig.node.json +26 -0
- data/gui/vite.config.ts +8 -0
- data/gui/vitest.config.ts +13 -0
- data/lib/ruby_reactor/async_router.rb +12 -8
- data/lib/ruby_reactor/context.rb +35 -9
- data/lib/ruby_reactor/context_serializer.rb +15 -0
- data/lib/ruby_reactor/dependency_graph.rb +2 -0
- data/lib/ruby_reactor/dsl/compose_builder.rb +8 -0
- data/lib/ruby_reactor/dsl/interrupt_builder.rb +48 -0
- data/lib/ruby_reactor/dsl/interrupt_step_config.rb +21 -0
- data/lib/ruby_reactor/dsl/map_builder.rb +14 -2
- data/lib/ruby_reactor/dsl/reactor.rb +12 -0
- data/lib/ruby_reactor/dsl/step_builder.rb +4 -0
- data/lib/ruby_reactor/executor/compensation_manager.rb +60 -27
- data/lib/ruby_reactor/executor/graph_manager.rb +2 -0
- data/lib/ruby_reactor/executor/result_handler.rb +118 -39
- data/lib/ruby_reactor/executor/retry_manager.rb +12 -1
- data/lib/ruby_reactor/executor/step_executor.rb +38 -4
- data/lib/ruby_reactor/executor.rb +86 -13
- data/lib/ruby_reactor/interrupt_result.rb +20 -0
- data/lib/ruby_reactor/map/collector.rb +71 -35
- data/lib/ruby_reactor/map/dispatcher.rb +162 -0
- data/lib/ruby_reactor/map/element_executor.rb +62 -56
- data/lib/ruby_reactor/map/execution.rb +44 -4
- data/lib/ruby_reactor/map/helpers.rb +44 -6
- data/lib/ruby_reactor/map/result_enumerator.rb +105 -0
- data/lib/ruby_reactor/reactor.rb +187 -1
- data/lib/ruby_reactor/registry.rb +25 -0
- data/lib/ruby_reactor/sidekiq_workers/worker.rb +1 -1
- data/lib/ruby_reactor/step/compose_step.rb +22 -6
- data/lib/ruby_reactor/step/map_step.rb +78 -19
- data/lib/ruby_reactor/storage/adapter.rb +32 -0
- data/lib/ruby_reactor/storage/redis_adapter.rb +213 -11
- data/lib/ruby_reactor/template/dynamic_source.rb +32 -0
- data/lib/ruby_reactor/utils/code_extractor.rb +31 -0
- data/lib/ruby_reactor/version.rb +1 -1
- data/lib/ruby_reactor/web/api.rb +206 -0
- data/lib/ruby_reactor/web/application.rb +53 -0
- data/lib/ruby_reactor/web/config.ru +5 -0
- data/lib/ruby_reactor/web/public/assets/index-VdeLgH9k.js +19 -0
- data/lib/ruby_reactor/web/public/assets/index-_z-6BvuM.css +1 -0
- data/lib/ruby_reactor/web/public/index.html +14 -0
- data/lib/ruby_reactor/web/public/vite.svg +1 -0
- data/lib/ruby_reactor.rb +94 -28
- data/llms-full.txt +66 -0
- data/llms.txt +7 -0
- metadata +66 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 863c83e92cbfab470e39107c580ceabce444133973b723662a89e5bf0211c3ab
|
|
4
|
+
data.tar.gz: 1209b5587efc055bc787731694a6dc2b7eab8b3d5a6ff729728aa29ea601e2e0
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 146d14cd32b12c95764e493f2de2d826dbe074ec4e4b44613b788d748d52fe38644fd86f1590f6bb0b1a0d1840c2200ab29a6d8e78db27f8f6abf099faf74756
|
|
7
|
+
data.tar.gz: 3d5b7c43182ee2d1c5c06ba90b7e2177a3dffbecdc8fb3e91db14f99bcd02807ebfa98c0e240412b790ec1d69609dc30995d9ee1aea9c8cb8ecd69d4d0fb2ac7
|
data/.rubocop.yml
CHANGED
|
@@ -13,6 +13,7 @@ AllCops:
|
|
|
13
13
|
- bin/*
|
|
14
14
|
- db/**/*
|
|
15
15
|
- config/**/*
|
|
16
|
+
- demo_app/**/*
|
|
16
17
|
|
|
17
18
|
Style/StringLiterals:
|
|
18
19
|
EnforcedStyle: double_quotes
|
|
@@ -27,13 +28,13 @@ Metrics/BlockLength:
|
|
|
27
28
|
Metrics/MethodLength:
|
|
28
29
|
Exclude:
|
|
29
30
|
- spec/**/*
|
|
30
|
-
Max:
|
|
31
|
+
Max: 36
|
|
31
32
|
CountComments: false
|
|
32
33
|
|
|
33
34
|
Metrics/ClassLength:
|
|
34
35
|
Exclude:
|
|
35
36
|
- spec/**/*
|
|
36
|
-
Max:
|
|
37
|
+
Max: 250
|
|
37
38
|
|
|
38
39
|
Style/Documentation:
|
|
39
40
|
Exclude:
|
|
@@ -96,3 +97,10 @@ RSpec/DescribeClass:
|
|
|
96
97
|
|
|
97
98
|
RSpec/AnyInstance:
|
|
98
99
|
Enabled: false
|
|
100
|
+
|
|
101
|
+
Naming/VariableNumber:
|
|
102
|
+
Enabled: false
|
|
103
|
+
|
|
104
|
+
Metrics/ModuleLength:
|
|
105
|
+
Max: 200
|
|
106
|
+
|
data/README.md
CHANGED
|
@@ -1,7 +1,20 @@
|
|
|
1
|
+
[](https://badge.fury.io/rb/ruby_reactor)
|
|
2
|
+
[](https://github.com/arturictus/ruby_reactor/actions) <!-- if you have CI -->
|
|
3
|
+
[](https://github.com/rubocop/rubocop)
|
|
4
|
+
[](https://rubygems.org/gems/ruby_reactor)
|
|
5
|
+
|
|
1
6
|
# RubyReactor
|
|
2
7
|
|
|
8
|
+
## Why Ruby Reactor?
|
|
9
|
+
|
|
10
|
+
Building complex business transactions often results in spaghetti code or brittle "god classes." Ruby Reactor solves this by implementing the **Saga Pattern** in a lightweight, developer-friendly package. It lets you define workflows as clear, dependency-driven steps without the boilerplate of heavy enterprise frameworks.
|
|
11
|
+
|
|
12
|
+
The key value is **Reliability**: if any part of your workflow fails, Ruby Reactor automatically triggers compensation logic to undo previous steps, ensuring your system never ends up in a corrupted half-state. Whether you're coordinating microservices or monolith modules, you get atomic-like consistency with background processing built-in.
|
|
13
|
+
|
|
3
14
|
A dynamic, dependency-resolving saga orchestrator for Ruby. Ruby Reactor implements the Saga pattern with compensation-based error handling and DAG-based execution planning. It leverages **Sidekiq** for asynchronous execution and **Redis** for state persistence.
|
|
4
15
|
|
|
16
|
+

|
|
17
|
+
|
|
5
18
|
## Features
|
|
6
19
|
|
|
7
20
|
- **DAG-based Execution**: Steps are executed based on their dependencies, allowing for parallel execution of independent steps.
|
|
@@ -9,8 +22,52 @@ A dynamic, dependency-resolving saga orchestrator for Ruby. Ruby Reactor impleme
|
|
|
9
22
|
- **Map & Parallel Execution**: Iterate over collections in parallel with the `map` step, distributing work across multiple workers.
|
|
10
23
|
- **Retries**: Configurable retry logic for failed steps, with exponential backoff.
|
|
11
24
|
- **Compensation**: Automatic rollback of completed steps when a failure occurs.
|
|
25
|
+
- **Interrupts**: Pause and resume workflows to wait for external events (webhooks, user approvals).
|
|
12
26
|
- **Input Validation**: Integrated with `dry-validation` for robust input checking.
|
|
13
27
|
|
|
28
|
+
## Comparison
|
|
29
|
+
|
|
30
|
+
| Feature | Ruby Reactor | dry-transaction | Trailblazer | Custom Sidekiq Jobs |
|
|
31
|
+
|--------------------------|--------------|-----------------|-------------|---------------------|
|
|
32
|
+
| DAG/Parallel execution | Yes | No | Limited | Manual |
|
|
33
|
+
| Auto compensation/undo | Yes | No | Manual | Manual |
|
|
34
|
+
| Interrupts (pause/resume)| Yes | No | No | Manual |
|
|
35
|
+
| Built-in web dashboard | Yes | No | No | No |
|
|
36
|
+
| Async with Sidekiq | Yes | No | Limited | Yes |
|
|
37
|
+
|
|
38
|
+
## Real-World Use Cases
|
|
39
|
+
|
|
40
|
+
- **E-commerce Checkout**: Orchestrate inventory reservation, payment authorization, and shipping label generation. If payment fails, automatically release inventory and cancel the shipping request.
|
|
41
|
+
- **Data Import Pipelines**: Ingest optional massive CSVs using `map` steps to validate and upsert records in parallel. If data validation fails for a chunk, fail fast or collect errors while letting valid chunks succeed.
|
|
42
|
+
- **Subscription Billing**: Coordinate Stripe charges, invoice email generation, and internal entitlement updates. Use interrupts to pause the workflow when 3rd-party APIs are required to continue the workflow or when specific customer approval is needed.
|
|
43
|
+
|
|
44
|
+
## Table of Contents
|
|
45
|
+
- [Features](#features)
|
|
46
|
+
- [Comparison](#comparison)
|
|
47
|
+
- [Real-World Use Cases](#real-world-use-cases)
|
|
48
|
+
- [Installation](#installation)
|
|
49
|
+
- [Configuration](#configuration)
|
|
50
|
+
- [Quick Start](#quick-start)
|
|
51
|
+
- [Web Dashboard](#web-dashboard)
|
|
52
|
+
- [Rails Installation](#rails-installation)
|
|
53
|
+
- [Usage](#usage)
|
|
54
|
+
- [Basic Example: User Registration](#basic-example-user-registration)
|
|
55
|
+
- [Async Execution](#async-execution)
|
|
56
|
+
- [Full Reactor Async](#full-reactor-async)
|
|
57
|
+
- [Step-Level Async](#step-level-async)
|
|
58
|
+
- [Interrupts (Pause & Resume)](#interrupts-pause--resume)
|
|
59
|
+
- [Map & Parallel Execution](#map--parallel-execution)
|
|
60
|
+
- [Map with Dynamic Source (ActiveRecord)](#map-with-dynamic-source-activerecord)
|
|
61
|
+
- [Input Validation](#input-validation)
|
|
62
|
+
- [Complex Workflows with Dependencies](#complex-workflows-with-dependencies)
|
|
63
|
+
- [Error Handling and Compensation](#error-handling-and-compensation)
|
|
64
|
+
- [Using Pre-defined Schemas](#using-pre-defined-schemas)
|
|
65
|
+
- [Documentation](#documentation)
|
|
66
|
+
- [Future improvements](#future-improvements)
|
|
67
|
+
- [Development](#development)
|
|
68
|
+
- [Contributing](#contributing)
|
|
69
|
+
- [Code of Conduct](#code-of-conduct)
|
|
70
|
+
|
|
14
71
|
## Installation
|
|
15
72
|
|
|
16
73
|
Add this line to your application's Gemfile:
|
|
@@ -47,6 +104,40 @@ RubyReactor.configure do |config|
|
|
|
47
104
|
end
|
|
48
105
|
```
|
|
49
106
|
|
|
107
|
+
|
|
108
|
+
## Quick Start
|
|
109
|
+
|
|
110
|
+
```ruby
|
|
111
|
+
class HelloReactor < RubyReactor::Reactor
|
|
112
|
+
step :greet do
|
|
113
|
+
run { Success("Hello from Ruby Reactor!") }
|
|
114
|
+
end
|
|
115
|
+
returns :greet
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
result = HelloReactor.run
|
|
119
|
+
puts result.value # => "Hello from Ruby Reactor!"
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## Web Dashboard
|
|
123
|
+
|
|
124
|
+
RubyReactor comes with a built-in web dashboard to inspect reactor executions, view logs, and retry failed steps.
|
|
125
|
+
|
|
126
|
+
### Rails Installation
|
|
127
|
+
|
|
128
|
+
Mount the dashboard engine in your `config/routes.rb`:
|
|
129
|
+
|
|
130
|
+
```ruby
|
|
131
|
+
Rails.application.routes.draw do
|
|
132
|
+
# ... other routes
|
|
133
|
+
mount RubyReactor::Web::Application => '/ruby_reactor'
|
|
134
|
+
end
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+

|
|
138
|
+
|
|
139
|
+
You can secure the dashboard using standard Rails authentication methods (e.g., `authenticate` block with Devise).
|
|
140
|
+
|
|
50
141
|
## Usage
|
|
51
142
|
|
|
52
143
|
RubyReactor allows you to define complex workflows as "reactors" with steps that can depend on each other, handle failures with compensations, and validate inputs.
|
|
@@ -198,6 +289,42 @@ def create(params)
|
|
|
198
289
|
end
|
|
199
290
|
```
|
|
200
291
|
|
|
292
|
+
### Interrupts (Pause & Resume)
|
|
293
|
+
|
|
294
|
+
Pause execution to wait for external events like webhooks or user approvals.
|
|
295
|
+
|
|
296
|
+
```ruby
|
|
297
|
+
class ApprovalReactor < RubyReactor::Reactor
|
|
298
|
+
step :submit_request do
|
|
299
|
+
run { |args| RequestService.submit(args) }
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
interrupt :wait_for_manager do
|
|
303
|
+
wait_for :submit_request
|
|
304
|
+
# Resume using this ID
|
|
305
|
+
correlation_id { |ctx| "req-#{ctx.result(:submit_request)[:id]}" }
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
step :process_decision do
|
|
309
|
+
argument :decision, result(:wait_for_manager)
|
|
310
|
+
run do |args|
|
|
311
|
+
args[:decision] == 'approved' ? Success() : Failure("Rejected")
|
|
312
|
+
end
|
|
313
|
+
end
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
# Usage:
|
|
317
|
+
# 1. Start execution
|
|
318
|
+
execution = ApprovalReactor.run(params) # => Returns Paused status
|
|
319
|
+
|
|
320
|
+
# 2. Later, resume it via correlation ID
|
|
321
|
+
ApprovalReactor.continue_by_correlation_id(
|
|
322
|
+
correlation_id: "req-123",
|
|
323
|
+
payload: "approved",
|
|
324
|
+
step_name: :wait_for_manager
|
|
325
|
+
)
|
|
326
|
+
```
|
|
327
|
+
|
|
201
328
|
### Map & Parallel Execution
|
|
202
329
|
|
|
203
330
|
Process collections in parallel using the `map` step:
|
|
@@ -223,6 +350,42 @@ class DataProcessingReactor < RubyReactor::Reactor
|
|
|
223
350
|
end
|
|
224
351
|
```
|
|
225
352
|
|
|
353
|
+
By using `async true` with `batch_size`, the system applies **Back Pressure** to efficiently manage resources. [Read more about Back Pressure & Resource Management](documentation/data_pipelines.md#back-pressure--resource-management).
|
|
354
|
+
|
|
355
|
+
#### Map with Dynamic Source (ActiveRecord)
|
|
356
|
+
|
|
357
|
+
You can use a block for `source` to dynamically fetch data, such as from ActiveRecord queries. The result is wrapped in a `ResultEnumerator` for easy access to successes and failures.
|
|
358
|
+
|
|
359
|
+
```ruby
|
|
360
|
+
map :archive_old_users do
|
|
361
|
+
argument :days, input(:days)
|
|
362
|
+
|
|
363
|
+
# Dynamic source using ActiveRecord
|
|
364
|
+
source do |args|
|
|
365
|
+
User.where("last_login_at < ?", args[:days].days.ago)
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
argument :user, element(:archive_old_users)
|
|
369
|
+
async true, batch_size: 100
|
|
370
|
+
|
|
371
|
+
step :archive do
|
|
372
|
+
argument :user, input(:user)
|
|
373
|
+
run { |args| args[:user].archive! }
|
|
374
|
+
end
|
|
375
|
+
|
|
376
|
+
returns :archive
|
|
377
|
+
end
|
|
378
|
+
|
|
379
|
+
step :summary do
|
|
380
|
+
argument :results, result(:archive_old_users)
|
|
381
|
+
run do |args|
|
|
382
|
+
puts "Archived: #{args[:results].successes.count}"
|
|
383
|
+
puts "Failed: #{args[:results].failures.count}"
|
|
384
|
+
Success()
|
|
385
|
+
end
|
|
386
|
+
end
|
|
387
|
+
```
|
|
388
|
+
|
|
226
389
|
### Input Validation
|
|
227
390
|
|
|
228
391
|
RubyReactor integrates with dry-validation for input validation:
|
|
@@ -540,6 +703,9 @@ Master the `map` feature for processing collections. Learn about parallel execut
|
|
|
540
703
|
### [Retry Configuration](documentation/retry_configuration.md)
|
|
541
704
|
Configure robust retry policies for your steps. This guide details the available backoff strategies (exponential, linear, fixed), how to configure retries at the reactor or step level, and how async retries work without blocking workers.
|
|
542
705
|
|
|
706
|
+
### [Interrupts](documentation/interrupts.md)
|
|
707
|
+
Learn how to pause and resume reactors to handle long-running processes, manual approvals, and asynchronous callbacks. Patterns for correlation IDs, timeouts, and payload validation.
|
|
708
|
+
|
|
543
709
|
### Examples
|
|
544
710
|
- [Order Processing](documentation/examples/order_processing.md) - Complete order processing workflow example
|
|
545
711
|
- [Payment Processing](documentation/examples/payment_processing.md) - Payment handling with compensation
|
|
@@ -548,12 +714,20 @@ Configure robust retry policies for your steps. This guide details the available
|
|
|
548
714
|
## Future improvements
|
|
549
715
|
|
|
550
716
|
- [X] Global id to serialize ActiveRecord classes
|
|
551
|
-
- [
|
|
552
|
-
- [ ] Descriptive errors
|
|
717
|
+
- [X] Descriptive errors
|
|
553
718
|
- [X] `map` step to iterate over arrays in parallel
|
|
554
719
|
- [X] `compose` special step to execute reactors as step
|
|
720
|
+
- [X] `interrupt` to pause and resume reactors
|
|
721
|
+
- [ ] Middlewares
|
|
555
722
|
- [ ] Async ruby to parallelize same level steps
|
|
556
|
-
- [
|
|
723
|
+
- [x] Web dashboard to inspect reactor results and errors
|
|
724
|
+
- [ ] Multiple storage adapters
|
|
725
|
+
- [X] Redis
|
|
726
|
+
- [ ] ActiveRecord
|
|
727
|
+
- [ ] Multiple Async adapters
|
|
728
|
+
- [X] Sidekiq
|
|
729
|
+
- [ ] ActiveJob
|
|
730
|
+
- [ ] OpenTelemetry support
|
|
557
731
|
|
|
558
732
|
## Development
|
|
559
733
|
|
data/Rakefile
CHANGED
|
@@ -5,6 +5,31 @@ require "rspec/core/rake_task"
|
|
|
5
5
|
|
|
6
6
|
RSpec::Core::RakeTask.new(:spec)
|
|
7
7
|
|
|
8
|
+
namespace :build do
|
|
9
|
+
desc "Build the UI assets"
|
|
10
|
+
task :ui do
|
|
11
|
+
puts "Building UI..."
|
|
12
|
+
system("cd gui && npm install && npm run build") || abort("UI build failed")
|
|
13
|
+
|
|
14
|
+
# Copy assets to public
|
|
15
|
+
# Vite builds to dist by default. We want it in lib/ruby_reactor/web/public
|
|
16
|
+
# Actually, we should configure Vite to build to the right place or copy it.
|
|
17
|
+
# Let's copy.
|
|
18
|
+
FileUtils.rm_rf("lib/ruby_reactor/web/public")
|
|
19
|
+
FileUtils.mkdir_p("lib/ruby_reactor/web/public")
|
|
20
|
+
FileUtils.cp_r("gui/dist/.", "lib/ruby_reactor/web/public/")
|
|
21
|
+
puts "UI built and assets copied to lib/ruby_reactor/web/public"
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
namespace :server do
|
|
26
|
+
desc "Start the server"
|
|
27
|
+
task :start do
|
|
28
|
+
puts "Starting server..."
|
|
29
|
+
system("rackup -Ilib lib/ruby_reactor/web/config.ru -p 9292") || abort("Server failed to start")
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
8
33
|
require "rubocop/rake_task"
|
|
9
34
|
|
|
10
35
|
RuboCop::RakeTask.new
|
|
@@ -43,53 +43,35 @@ class UserTransformationReactor < RubyReactor::Reactor
|
|
|
43
43
|
end
|
|
44
44
|
```
|
|
45
45
|
|
|
46
|
-
##
|
|
46
|
+
## Dynamic Sources & ActiveRecord
|
|
47
47
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
To enable async execution, simply add the `async true` directive to your map definition.
|
|
48
|
+
The `map` step supports a dynamic `source` block, which is particularly useful when working with ActiveRecord or when the collection depends on input arguments. Instead of passing a static collection, you can define a block that returns an Enumerable or an `ActiveRecord::Relation`.
|
|
51
49
|
|
|
52
50
|
```ruby
|
|
53
|
-
map :
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
argument :order, input(:order)
|
|
62
|
-
run { PaymentService.charge(args[:order]) }
|
|
51
|
+
map :process_products do
|
|
52
|
+
argument :filter, input(:filter)
|
|
53
|
+
|
|
54
|
+
# Dynamic source block
|
|
55
|
+
source do |args|
|
|
56
|
+
# This block executes at runtime
|
|
57
|
+
threshold = args[:filter][:stock]
|
|
58
|
+
Product.where("stock >= ?", threshold)
|
|
63
59
|
end
|
|
64
60
|
|
|
65
|
-
|
|
61
|
+
argument :product, element(:process_products)
|
|
62
|
+
async true, batch_size: 100
|
|
63
|
+
|
|
64
|
+
step :process do
|
|
65
|
+
# ...
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
returns :process
|
|
66
69
|
end
|
|
67
70
|
```
|
|
68
71
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
```mermaid
|
|
72
|
-
sequenceDiagram
|
|
73
|
-
participant Reactor
|
|
74
|
-
participant Redis
|
|
75
|
-
participant Sidekiq
|
|
76
|
-
participant Worker
|
|
77
|
-
|
|
78
|
-
Reactor->>Redis: Store Context
|
|
79
|
-
Reactor->>Sidekiq: Enqueue MapElementWorkers
|
|
80
|
-
Note over Reactor: Returns AsyncResult immediately
|
|
81
|
-
|
|
82
|
-
loop For each element
|
|
83
|
-
Sidekiq->>Worker: Process Element
|
|
84
|
-
Worker->>Redis: Update Element Result
|
|
85
|
-
end
|
|
86
|
-
|
|
87
|
-
Worker->>Sidekiq: Enqueue MapCollectorWorker (when done)
|
|
88
|
-
Sidekiq->>Worker: Run Collector
|
|
89
|
-
Worker->>Redis: Store Final Result
|
|
90
|
-
```
|
|
72
|
+
When an `ActiveRecord::Relation` is returned, RubyReactor efficiently batches the query using database-level `OFFSET` and `LIMIT` based on the configured `batch_size`, preventing memory bloat by not loading all records at once.
|
|
91
73
|
|
|
92
|
-
## Batch Processing
|
|
74
|
+
## Batch Processing Mechanism
|
|
93
75
|
|
|
94
76
|
When processing large datasets asynchronously, you can control the parallelism using `batch_size`. This limits how many Sidekiq jobs are enqueued simultaneously, preventing system overload.
|
|
95
77
|
|
|
@@ -107,10 +89,42 @@ map :bulk_import do
|
|
|
107
89
|
end
|
|
108
90
|
```
|
|
109
91
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
92
|
+
### Back Pressure & Resource Management
|
|
93
|
+
|
|
94
|
+
When `async true` is used with a `batch_size`, RubyReactor implements an intelligent **back pressure** mechanism. Instead of flooding Redis and Sidekiq with millions of jobs immediately (which is the standard behavior for many background job systems), the system processes data in controlled chunks.
|
|
95
|
+
|
|
96
|
+
This approach provides critical benefits for stability and scalability:
|
|
97
|
+
|
|
98
|
+
1. **Memory Efficiency**: By using `ActiveRecord` batching (`LIMIT` / `OFFSET`), only the current batch of records is loaded into memory. This allows processing datasets larger than available RAM.
|
|
99
|
+
2. **Redis Protection**: Prevents "Queue Explosion". Only a small number of job arguments are stored in Redis at any time, preventing OOM errors in your Redis instance.
|
|
100
|
+
3. **Database Stability**: Database load is distributed over time rather than spiking all at once.
|
|
101
|
+
|
|
102
|
+
**Visualizing the Flow:**
|
|
103
|
+
|
|
104
|
+
```mermaid
|
|
105
|
+
graph TD
|
|
106
|
+
Start[Start Map] -->|Batch Size: N| BatchManager
|
|
107
|
+
|
|
108
|
+
subgraph "Back Pressure Loop"
|
|
109
|
+
BatchManager[Batch Manager] -->|Fetch N Items| DB[(Database)]
|
|
110
|
+
DB --> Records
|
|
111
|
+
Records -->|Enqueue N Jobs| Sidekiq
|
|
112
|
+
|
|
113
|
+
Sidekiq --> W1[Worker 1]
|
|
114
|
+
Sidekiq --> W2[Worker 2]
|
|
115
|
+
|
|
116
|
+
W1 -.->|Complete| Check{Batch Done?}
|
|
117
|
+
W2 -.->|Complete| Check
|
|
118
|
+
|
|
119
|
+
Check -->|No| Wait[Wait]
|
|
120
|
+
Check -->|Yes| Next[Trigger Next Batch]
|
|
121
|
+
Next --> BatchManager
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
BatchManager -->|No More Items| Finish[Aggregator]
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
This ensures that the system works at the speed of your workers, not the speed of the enqueueing process, maintaining a constant and manageable resource footprint regardless of dataset size.
|
|
114
128
|
|
|
115
129
|
## Error Handling
|
|
116
130
|
|
|
@@ -128,9 +142,9 @@ map :strict_processing do
|
|
|
128
142
|
end
|
|
129
143
|
```
|
|
130
144
|
|
|
131
|
-
### Collecting
|
|
145
|
+
### Collecting Results (Successes & Failures)
|
|
132
146
|
|
|
133
|
-
If you want to process all elements regardless of failures, set `fail_fast false`.
|
|
147
|
+
If you want to process all elements regardless of failures, set `fail_fast false`. The map step returns a `ResultEnumerator` that allows you to easily separate successful executions from failures.
|
|
134
148
|
|
|
135
149
|
```ruby
|
|
136
150
|
map :resilient_processing do
|
|
@@ -145,18 +159,39 @@ map :resilient_processing do
|
|
|
145
159
|
end
|
|
146
160
|
|
|
147
161
|
returns :risky_operation
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
step :analyze_results do
|
|
165
|
+
argument :results, result(:resilient_processing)
|
|
166
|
+
|
|
167
|
+
run do |args|
|
|
168
|
+
col = args[:results]
|
|
169
|
+
|
|
170
|
+
# Iterate over successful results
|
|
171
|
+
col.successes.each do |value|
|
|
172
|
+
# 'value' is the direct return value of the map element
|
|
173
|
+
puts "Success: #{value}"
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# Iterate over failures
|
|
177
|
+
col.failures.each do |error|
|
|
178
|
+
# 'error' is the failure object/message itself
|
|
179
|
+
puts "Error: #{error}"
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# Note: Iterating the collection directly yields wrapped objects
|
|
183
|
+
col.each do |result|
|
|
184
|
+
if result.is_a?(RubyReactor::Success)
|
|
185
|
+
puts "Wrapped Value: #{result.value}"
|
|
186
|
+
else
|
|
187
|
+
puts "Wrapped Error: #{result.error}"
|
|
188
|
+
end
|
|
189
|
+
end
|
|
148
190
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
failed = results.select(&:failure?).map(&:error)
|
|
154
|
-
|
|
155
|
-
{
|
|
156
|
-
processed: successful,
|
|
157
|
-
errors: failed,
|
|
158
|
-
success_rate: successful.length.to_f / results.length
|
|
159
|
-
}
|
|
191
|
+
Success({
|
|
192
|
+
success_count: col.successes.count,
|
|
193
|
+
failure_count: col.failures.count
|
|
194
|
+
})
|
|
160
195
|
end
|
|
161
196
|
end
|
|
162
197
|
```
|
|
@@ -192,33 +227,4 @@ end
|
|
|
192
227
|
- **Async Mode**: Retries are handled by requeuing the Sidekiq job with a delay. This is non-blocking and efficient.
|
|
193
228
|
- **Sync Mode**: Retries happen immediately within the execution thread (blocking).
|
|
194
229
|
|
|
195
|
-
## Visualization
|
|
196
230
|
|
|
197
|
-
### Async Batch Execution
|
|
198
|
-
|
|
199
|
-
```mermaid
|
|
200
|
-
graph TD
|
|
201
|
-
Start[Start Map] --> Init[Initialize Batch]
|
|
202
|
-
Init --> Q1["Queue Initial Batch<br/>(Size N)"]
|
|
203
|
-
|
|
204
|
-
subgraph Workers
|
|
205
|
-
W1[Worker 1]
|
|
206
|
-
W2[Worker 2]
|
|
207
|
-
W3[Worker ...]
|
|
208
|
-
end
|
|
209
|
-
|
|
210
|
-
Q1 --> W1
|
|
211
|
-
Q1 --> W2
|
|
212
|
-
|
|
213
|
-
W1 -->|Complete| Next1{More Items?}
|
|
214
|
-
W2 -->|Complete| Next2{More Items?}
|
|
215
|
-
|
|
216
|
-
Next1 -->|Yes| Q2[Queue Next Item]
|
|
217
|
-
Next2 -->|Yes| Q2
|
|
218
|
-
|
|
219
|
-
Q2 --> W3
|
|
220
|
-
|
|
221
|
-
Next1 -->|No| Check{All Done?}
|
|
222
|
-
Check -->|Yes| Collect[Run Collector]
|
|
223
|
-
Collect --> Finish[Final Result]
|
|
224
|
-
```
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
# Interrupts (Pause & Resume)
|
|
2
|
+
|
|
3
|
+
RubyReactor introduces the `interrupt` mechanism to support long-running processes that require external input, such as user approvals, webhooks, or asynchronous job completions. Unlike standard steps that execute immediately, an `interrupt` pauses the reactor execution and persists its state, waiting for a signal to resume.
|
|
4
|
+
|
|
5
|
+
## DSL Usage
|
|
6
|
+
|
|
7
|
+
Use the `interrupt` keyword to define a pause point in your reactor.
|
|
8
|
+
|
|
9
|
+
```ruby
|
|
10
|
+
class ReportReactor < RubyReactor::Reactor
|
|
11
|
+
step :request_report do
|
|
12
|
+
run do
|
|
13
|
+
response = HTTP.post("https://api.example.com/reports")
|
|
14
|
+
Success(response.fetch(:id))
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
interrupt :wait_for_report do
|
|
19
|
+
# Declare dependency: execution must trigger this interrupt only after :request_report succeeds
|
|
20
|
+
wait_for :request_report
|
|
21
|
+
|
|
22
|
+
# Optional: deterministic correlation ID for looking up this execution later
|
|
23
|
+
correlation_id do |context|
|
|
24
|
+
"report-#{context.result(:request_report)}"
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Optional: timeout in seconds
|
|
28
|
+
# Strategies:
|
|
29
|
+
# - :lazy (default) - checked only when resume is attempted
|
|
30
|
+
# - :active - schedules a background job to wake up the reactor and fail it
|
|
31
|
+
timeout 1800, strategy: :active
|
|
32
|
+
|
|
33
|
+
# Optional: validate incoming payload immediately using dry-validation
|
|
34
|
+
validate do
|
|
35
|
+
required(:status).filled(:string)
|
|
36
|
+
required(:url).filled(:string)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Optional: limit validation attempts (default: 1)
|
|
40
|
+
# If exhausted, the reactor is cancelled and compensated.
|
|
41
|
+
# Use :infinity for unlimited attempts.
|
|
42
|
+
max_attempts 3
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
step :process_report do
|
|
46
|
+
# The result of the interrupt step is the payload provided when resuming
|
|
47
|
+
argument :webhook_payload, result(:wait_for_report)
|
|
48
|
+
|
|
49
|
+
run do |args|
|
|
50
|
+
Success(ReportProcessor.call(args[:webhook_payload]))
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
### Options
|
|
57
|
+
|
|
58
|
+
* **`wait_for`**: declare dependencies similar to `step`.
|
|
59
|
+
* **`correlation_id`**: A block that returns a unique string to identify this execution. This allows you to resume the reactor using a business key (e.g., order ID) instead of the internal execution UUID.
|
|
60
|
+
* **`timeout`**: Set a time limit for the interrupt.
|
|
61
|
+
* **`validate`**: A `dry-validation` schema block to validate the payload provided when resuming.
|
|
62
|
+
* **`max_attempts`**: Limit the number of times `continue` can be called with an invalid payload before the reactor is automatically cancelled and compensated. Defaults to 1. Set to `:infinity` for unlimited retries.
|
|
63
|
+
|
|
64
|
+
## Runtime Behavior
|
|
65
|
+
|
|
66
|
+
When a reactor encounters an `interrupt`:
|
|
67
|
+
|
|
68
|
+
1. It executes any dependencies.
|
|
69
|
+
2. It persists the full `Context` (results of previous steps) to the configured storage (e.g., Redis).
|
|
70
|
+
3. It returns an `InterruptResult` and halts execution.
|
|
71
|
+
|
|
72
|
+
```ruby
|
|
73
|
+
execution = ReportReactor.run(company_id: 1)
|
|
74
|
+
|
|
75
|
+
if execution.paused?
|
|
76
|
+
execution.id # => "uuid-123"
|
|
77
|
+
execution.status # => :paused
|
|
78
|
+
end
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## Resuming Execution
|
|
82
|
+
|
|
83
|
+
You can resume a paused reactor using its UUID or the defined `correlation_id`.
|
|
84
|
+
|
|
85
|
+
### By UUID
|
|
86
|
+
|
|
87
|
+
```ruby
|
|
88
|
+
ReportReactor.continue(
|
|
89
|
+
id: "uuid-123",
|
|
90
|
+
payload: { status: "completed", url: "..." },
|
|
91
|
+
step_name: :wait_for_report
|
|
92
|
+
)
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### By Correlation ID
|
|
96
|
+
|
|
97
|
+
```ruby
|
|
98
|
+
ReportReactor.continue_by_correlation_id(
|
|
99
|
+
correlation_id: "report-999",
|
|
100
|
+
payload: { status: "completed", url: "..." },
|
|
101
|
+
step_name: :wait_for_report
|
|
102
|
+
)
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
### Resuming Method Styles
|
|
106
|
+
|
|
107
|
+
There are two ways to invoke continuation:
|
|
108
|
+
|
|
109
|
+
1. **Strict / Fire-and-Forget (Class Method)**:
|
|
110
|
+
* `Reactor.continue(...)`
|
|
111
|
+
* If payload is invalid, it **automatically compensates (undo)** and cancels the reactor.
|
|
112
|
+
* Best for webhooks where you can't ask the sender to fix the payload.
|
|
113
|
+
|
|
114
|
+
2. **Flexible (Instance Method)**:
|
|
115
|
+
* First find the reactor: `reactor = ReportReactor.find("uuid-123")`
|
|
116
|
+
* Then call: `result = reactor.continue(payload: ..., step_name: :wait_for_report)`
|
|
117
|
+
* If payload is invalid, it returns a failure result but **does not** cancel execution.
|
|
118
|
+
* Allows you to handle the error (e.g., show a form error to a user) and try again.
|
|
119
|
+
|
|
120
|
+
## Cancellation & Undo
|
|
121
|
+
|
|
122
|
+
You can cancel a paused reactor if the operation is no longer needed.
|
|
123
|
+
|
|
124
|
+
```ruby
|
|
125
|
+
# Undo: Runs defined compensations for completed steps in reverse order, then deletes execution.
|
|
126
|
+
ReportReactor.undo("uuid-123")
|
|
127
|
+
|
|
128
|
+
# Cancel: Stops execution immediately and marks the reactor as cancelled with the provided reason.
|
|
129
|
+
# The context is preserved for inspection, but resumption is disabled.
|
|
130
|
+
ReportReactor.cancel("uuid-123", reason: "User cancelled")
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
## Common Use Cases
|
|
134
|
+
|
|
135
|
+
### Human Approvals
|
|
136
|
+
|
|
137
|
+
```ruby
|
|
138
|
+
interrupt :wait_for_approval do
|
|
139
|
+
wait_for :submit_request
|
|
140
|
+
correlation_id { |ctx| "approval-#{ctx.input(:request_id)}" }
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
step :process_decision do
|
|
144
|
+
argument :decision, result(:wait_for_approval)
|
|
145
|
+
run do |args|
|
|
146
|
+
if args[:decision][:approved]
|
|
147
|
+
Success("Approved")
|
|
148
|
+
else
|
|
149
|
+
Failure("Rejected")
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
### Webhooks
|
|
156
|
+
|
|
157
|
+
Use `correlation_id` to map an external resource ID (like a Payment Intent ID) back to the reactor waiting for confirmation.
|
|
158
|
+
|
|
159
|
+
### Scheduled Follow-ups
|
|
160
|
+
|
|
161
|
+
Using `timeout` with `strategy: :active` to wake up a reactor after a delay if no external event occurs (e.g., expiring a reservation).
|