stepper_motor 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +12 -0
- data/.github/workflows/ci.yml +51 -0
- data/CHANGELOG.md +77 -2
- data/Gemfile +11 -0
- data/README.md +13 -374
- data/Rakefile +21 -3
- data/bin/test +5 -0
- data/lib/generators/install_generator.rb +6 -1
- data/lib/generators/stepper_motor_migration_003.rb.erb +6 -0
- data/lib/generators/stepper_motor_migration_004.rb.erb +26 -0
- data/lib/stepper_motor/forward_scheduler.rb +8 -4
- data/lib/stepper_motor/journey/flow_control.rb +58 -0
- data/lib/stepper_motor/journey/recovery.rb +34 -0
- data/lib/stepper_motor/journey.rb +85 -84
- data/lib/stepper_motor/perform_step_job_v2.rb +2 -2
- data/lib/stepper_motor/recover_stuck_journeys_job_v1.rb +3 -1
- data/lib/stepper_motor/step.rb +70 -5
- data/lib/stepper_motor/version.rb +1 -1
- data/lib/stepper_motor.rb +0 -1
- data/lib/tasks/stepper_motor_tasks.rake +8 -0
- data/manual/MANUAL.md +538 -0
- data/rbi/stepper_motor.rbi +459 -0
- data/sig/stepper_motor.rbs +406 -3
- data/stepper_motor.gemspec +49 -0
- data/test/dummy/Rakefile +8 -0
- data/test/dummy/app/assets/stylesheets/application.css +1 -0
- data/test/dummy/app/controllers/application_controller.rb +6 -0
- data/test/dummy/app/helpers/application_helper.rb +4 -0
- data/test/dummy/app/jobs/application_job.rb +9 -0
- data/test/dummy/app/mailers/application_mailer.rb +6 -0
- data/test/dummy/app/models/application_record.rb +5 -0
- data/test/dummy/app/views/layouts/application.html.erb +27 -0
- data/test/dummy/app/views/layouts/mailer.html.erb +13 -0
- data/test/dummy/app/views/layouts/mailer.text.erb +1 -0
- data/test/dummy/app/views/pwa/manifest.json.erb +22 -0
- data/test/dummy/app/views/pwa/service-worker.js +26 -0
- data/test/dummy/bin/dev +2 -0
- data/test/dummy/bin/rails +4 -0
- data/test/dummy/bin/rake +4 -0
- data/test/dummy/bin/setup +34 -0
- data/test/dummy/config/application.rb +28 -0
- data/test/dummy/config/boot.rb +7 -0
- data/test/dummy/config/cable.yml +10 -0
- data/test/dummy/config/database.yml +32 -0
- data/test/dummy/config/environment.rb +7 -0
- data/test/dummy/config/environments/development.rb +71 -0
- data/test/dummy/config/environments/production.rb +91 -0
- data/test/dummy/config/environments/test.rb +55 -0
- data/test/dummy/config/initializers/content_security_policy.rb +27 -0
- data/test/dummy/config/initializers/filter_parameter_logging.rb +10 -0
- data/test/dummy/config/initializers/inflections.rb +18 -0
- data/test/dummy/config/initializers/stepper_motor.rb +3 -0
- data/test/dummy/config/locales/en.yml +31 -0
- data/test/dummy/config/puma.rb +40 -0
- data/test/dummy/config/routes.rb +16 -0
- data/test/dummy/config/storage.yml +34 -0
- data/test/dummy/config.ru +8 -0
- data/test/dummy/db/migrate/20250520094921_stepper_motor_migration_001.rb +38 -0
- data/test/dummy/db/migrate/20250520094922_stepper_motor_migration_002.rb +8 -0
- data/test/dummy/db/migrate/20250522212312_stepper_motor_migration_003.rb +7 -0
- data/test/dummy/db/migrate/20250525110812_stepper_motor_migration_004.rb +28 -0
- data/test/dummy/db/schema.rb +37 -0
- data/test/dummy/public/400.html +114 -0
- data/test/dummy/public/404.html +114 -0
- data/test/dummy/public/406-unsupported-browser.html +114 -0
- data/test/dummy/public/422.html +114 -0
- data/test/dummy/public/500.html +114 -0
- data/test/dummy/public/icon.png +0 -0
- data/test/dummy/public/icon.svg +3 -0
- data/test/side_effects_helper.rb +67 -0
- data/test/stepper_motor/cyclic_scheduler_test.rb +77 -0
- data/{spec/stepper_motor/forward_scheduler_spec.rb → test/stepper_motor/forward_scheduler_test.rb} +9 -10
- data/test/stepper_motor/journey/exception_handling_test.rb +89 -0
- data/test/stepper_motor/journey/flow_control_test.rb +78 -0
- data/test/stepper_motor/journey/idempotency_test.rb +65 -0
- data/test/stepper_motor/journey/step_definition_test.rb +187 -0
- data/test/stepper_motor/journey/uniqueness_test.rb +48 -0
- data/test/stepper_motor/journey_test.rb +352 -0
- data/{spec/stepper_motor/recover_stuck_journeys_job_spec.rb → test/stepper_motor/recover_stuck_journeys_job_test.rb} +14 -14
- data/{spec/stepper_motor/recovery_spec.rb → test/stepper_motor/recovery_test.rb} +27 -27
- data/test/stepper_motor/test_helper_test.rb +44 -0
- data/test/stepper_motor_test.rb +9 -0
- data/test/test_helper.rb +46 -0
- metadata +120 -24
- data/.rspec +0 -3
- data/.ruby-version +0 -1
- data/.standard.yml +0 -4
- data/.yardopts +0 -1
- data/spec/helpers/side_effects.rb +0 -85
- data/spec/spec_helper.rb +0 -90
- data/spec/stepper_motor/cyclic_scheduler_spec.rb +0 -68
- data/spec/stepper_motor/generator_spec.rb +0 -16
- data/spec/stepper_motor/journey_spec.rb +0 -401
- data/spec/stepper_motor/test_helper_spec.rb +0 -48
- data/spec/stepper_motor_spec.rb +0 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 90e48cc6e234e073f0582319748da22480e8c3fa2bb81b8217faea879ce542c5
|
4
|
+
data.tar.gz: 02d2e302fa8238a3c32130e02a79d8f9b2b16b15c59713f996e57ebe03efee2c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1378f0b6baccf36921501e68e6c24fb26fe62ab13c07098e2af2333bf811b2306a3de3e77c707fef3b7656b7740836e07079f93b48bca873906d62e87a576b5e
|
7
|
+
data.tar.gz: 72f06897646794459f789d51bed060472158755445216df2b9dea8d25a835eaad5692946f77fa366754a9b14382958b2497f018739fe0bd69104bb0fee766797
|
@@ -0,0 +1,51 @@
|
|
1
|
+
name: CI
|
2
|
+
|
3
|
+
on:
|
4
|
+
pull_request:
|
5
|
+
push:
|
6
|
+
branches: [ main ]
|
7
|
+
|
8
|
+
jobs:
|
9
|
+
lint:
|
10
|
+
runs-on: ubuntu-latest
|
11
|
+
steps:
|
12
|
+
- name: Checkout code
|
13
|
+
uses: actions/checkout@v4
|
14
|
+
|
15
|
+
- name: Set up Ruby
|
16
|
+
uses: ruby/setup-ruby@v1
|
17
|
+
with:
|
18
|
+
ruby-version: 3.2.2
|
19
|
+
bundler-cache: true
|
20
|
+
|
21
|
+
- name: Lint code for consistent style
|
22
|
+
run: bundle exec standardrb
|
23
|
+
|
24
|
+
test:
|
25
|
+
runs-on: ubuntu-latest
|
26
|
+
|
27
|
+
# services:
|
28
|
+
# redis:
|
29
|
+
# image: redis
|
30
|
+
# ports:
|
31
|
+
# - 6379:6379
|
32
|
+
# options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5
|
33
|
+
steps:
|
34
|
+
- name: Install packages
|
35
|
+
run: sudo apt-get update && sudo apt-get install --no-install-recommends -y build-essential git libyaml-dev pkg-config google-chrome-stable
|
36
|
+
|
37
|
+
- name: Checkout code
|
38
|
+
uses: actions/checkout@v4
|
39
|
+
|
40
|
+
- name: Set up Ruby
|
41
|
+
uses: ruby/setup-ruby@v1
|
42
|
+
with:
|
43
|
+
ruby-version: 3.2.2
|
44
|
+
bundler-cache: true
|
45
|
+
|
46
|
+
- name: Run tests
|
47
|
+
env:
|
48
|
+
RAILS_ENV: test
|
49
|
+
# REDIS_URL: redis://localhost:6379/0
|
50
|
+
run: bin/test
|
51
|
+
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,78 @@
|
|
1
|
-
|
1
|
+
# Changelog
|
2
2
|
|
3
|
-
|
3
|
+
## [0.1.8] - 2025-05-25
|
4
|
+
|
5
|
+
- Add ability to pause and resume journeys (#24)
|
6
|
+
- Add basic exception rescue during steps (#23)
|
7
|
+
- Add idempotency keys when performing steps (#21)
|
8
|
+
- Add support for blockless step definitions (#22)
|
9
|
+
- Migrate from RSpec to Minitest (#19)
|
10
|
+
- Add Rake task for recovery
|
11
|
+
- Add proper Rails engine tests
|
12
|
+
- Improve test organization and coverage
|
13
|
+
- Add frozen string literals
|
14
|
+
- Relocate Recovery into a module
|
15
|
+
|
16
|
+
## [0.1.7] - 2025-05-19
|
17
|
+
|
18
|
+
- Improve Rails integration reliability:
|
19
|
+
- Load Railtie earlier in the boot process
|
20
|
+
- Fix generator loading to use relative paths instead of load paths
|
21
|
+
- Improve database migration for stuck journeys:
|
22
|
+
- Add concurrent index creation for better performance
|
23
|
+
- Fix index creation to work in all environments
|
24
|
+
|
25
|
+
## [0.1.6] - 2025-05-19
|
26
|
+
|
27
|
+
- Add functionality to recover journeys stuck in "performing" state
|
28
|
+
- Add `RecoverStuckJourneysJobV1` with two recovery modes:
|
29
|
+
- `:reattempt`: Tries to restart the step where the journey hung
|
30
|
+
- `:cancel`: Cancels the journey
|
31
|
+
- Add database index to assist with stuck journey recovery
|
32
|
+
- Add test coverage for recovery scenarios
|
33
|
+
- Add ability to configure recovery behavior per journey class
|
34
|
+
|
35
|
+
## [0.1.5] - 2025-03-17
|
36
|
+
|
37
|
+
- Refactor PerformStepJob to use Journey class in job arguments
|
38
|
+
- Remove GlobalID dependency
|
39
|
+
- Add ability to resolve Journey from base class using `find()`
|
40
|
+
- Prepare groundwork for future improvements (#12)
|
41
|
+
|
42
|
+
## [0.1.4] - 2025-03-11
|
43
|
+
|
44
|
+
- Fix critical bug with endless self-replication of PerformStepJobs
|
45
|
+
- Fix misnamed ActiveJob parameter that caused immediate job execution
|
46
|
+
- Add test to ensure proper parameter passing to ActiveJob
|
47
|
+
- Improve job scheduling reliability
|
48
|
+
|
49
|
+
## [0.1.3] - 2025-02-28
|
50
|
+
|
51
|
+
- Add test suite with journey behavior testing
|
52
|
+
- Add support for UUID foreign keys in migrations
|
53
|
+
- Add Rails engine integration with Railtie
|
54
|
+
- Add support for concurrent index creation
|
55
|
+
- Add error handling for database initialization
|
56
|
+
- Add support for eager loading in Rails applications
|
57
|
+
- Add logging with tagged contexts
|
58
|
+
- Add support for step reattempts and cancellations
|
59
|
+
- Add support for multiple journeys per hero with `allow_multiple` option
|
60
|
+
- Add state tracking with steps_entered and steps_completed counters
|
61
|
+
- Add support for both wait: and after: timing options in steps
|
62
|
+
- Add error handling for invalid step configurations
|
63
|
+
|
64
|
+
## [0.1.2] - 2025-01-01
|
65
|
+
|
66
|
+
- Fix Railtie integration
|
67
|
+
- Ensure proper Rails engine functionality
|
68
|
+
|
69
|
+
## [0.1.1] - 2025-01-01
|
70
|
+
|
71
|
+
- Update dependencies
|
72
|
+
- Improve compatibility with latest gem versions
|
73
|
+
|
74
|
+
## [0.1.0] - 2024-09-29
|
75
|
+
|
76
|
+
- Initial release
|
77
|
+
- Add basic stepper motor functionality
|
78
|
+
- Add Rails integration support
|
data/Gemfile
ADDED
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
#
|
1
|
+
# stepper_motor
|
2
2
|
|
3
3
|
Is a useful tool for running stepped or iterative workflows inside your Rails application.
|
4
4
|
|
@@ -24,54 +24,7 @@ end
|
|
24
24
|
SignupJourney.create!(hero: current_user)
|
25
25
|
```
|
26
26
|
|
27
|
-
|
28
|
-
|
29
|
-
Add the gem to the application's Gemfile, and then generate and run the migration
|
30
|
-
|
31
|
-
$ bundle add stepper_motor
|
32
|
-
$ bundle install
|
33
|
-
$ bin/rails g stepper_motor:install --uuid # Pass "uuid" if you are using UUID for your primary and foreign keys
|
34
|
-
$ bin/rails db:migrate
|
35
|
-
|
36
|
-
|
37
|
-
## Intro
|
38
|
-
|
39
|
-
`stepper_motor` solves a real, tangible problem in Rails apps - tracking activities over long periods of time. It does so in a durable, reentrant and consistent manner, utilizing the guarantees provided by your relational database you already have.
|
40
|
-
|
41
|
-
## Philosophy behind StepperMotor
|
42
|
-
|
43
|
-
Most of our applications have workflows which have to happen in steps. They pretty much always have some things in common:
|
44
|
-
|
45
|
-
* We want just one workflow of a certain type per user or per business transaction
|
46
|
-
* We want only one parallel execution of a unique workflow at a time
|
47
|
-
* We want the steps to be explicitly idempotent
|
48
|
-
* We want visibility into the step our workflow is in, what step it is going to enter, what step it has left
|
49
|
-
|
50
|
-
While Rails provides great abstractions for "inline" actions induced via APIs or web requests in the form of ActionController, and great abstractions for single "unit of work" tasks via ActiveJob - these are lacking if one wants true idempotency and correct state tracking throughout multiple steps. When a workflow like this has to be implemented in a system, the choice usually goes out to a number of possible solutions:
|
51
|
-
|
52
|
-
* Trying ActiveJob-specific "batch" workflows, such as [Sidekiq Pro's batches](https://github.com/sidekiq/sidekiq/wiki/Batches) or [good_job batches](https://github.com/bensheldon/good_job?tab=readme-ov-file#batches)
|
53
|
-
* State machines attached to an ActiveRecord, via tools like [aasm](https://github.com/aasm/aasm) or [state_machine_enum](https://github.com/cheddar-me/state_machine_enum) - locking and controlling transitions then usually falls on the developer
|
54
|
-
* Adopting a complex solution like [Temporal.io](https://temporal.io/), with the app relegated to just executing parts of the workflow
|
55
|
-
|
56
|
-
We believe all of these solutions do not quite hit the "sweet spot" where step workflows would integrate well with Rails.
|
57
|
-
|
58
|
-
* Most Rails apps already have a perfectly fit transactional, durable data store - the main database
|
59
|
-
* The devloper should not have intimate understanding of DB atomicity and ActiveRecord `with_lock` and `reload` to have step workflows
|
60
|
-
* It should not be necessary to configure a whole extra service (like Temporal.io) just for supporting those workflows. A service like that should be a part of your monolith, not an external application. It should not be necessary to talk to that service using complex, Ruby-unfriendly protocols and interfaces like gRPC.
|
61
|
-
|
62
|
-
So, StepperMotor aims to give you "just enough of Temporal-like functionality" for most Rails-bound workflows. Without extra dependencies, network calls, services or having to learn extra languages. So let's dive in.
|
63
|
-
|
64
|
-
## How we do it
|
65
|
-
|
66
|
-
StepperMotor is built around the concept of a `Journey`. A `Journey` [is a sequence of steps happening to a `hero`](https://en.wikipedia.org/wiki/Hero%27s_journey) - once launched, the journey will run until it either finishes or cancels. A `Journey` is just an `ActiveRecord` model, with all the persistence methods you already know and use.
|
67
|
-
|
68
|
-
Steps are defined inside the Journey as blocks, and they run in the context of the `Journey` model. The following constraints apply:
|
69
|
-
|
70
|
-
* For any one Journey, only one Fiber/Thread/Process may be performing a step on it
|
71
|
-
* For any one Journey, only one step can be executing at any given time
|
72
|
-
* For any `hero`, multiple different Journeys may exist and be in different stages of completion
|
73
|
-
|
74
|
-
The `step` blocks get executed in the context of the `Journey` model instance. This is done so that you can define helper methods in the `Journey` subclass, and make good use of them. A Journey links to just one record - the `hero`.
|
27
|
+
Want to know more? Dive into the [manual](file.MANUAL.html) we provide.
|
75
28
|
|
76
29
|
## Installation
|
77
30
|
|
@@ -79,336 +32,18 @@ Add the gem to the application's Gemfile, and then generate and run the migratio
|
|
79
32
|
|
80
33
|
$ bundle add stepper_motor
|
81
34
|
$ bundle install
|
82
|
-
$ bin/rails g stepper_motor:install
|
35
|
+
$ bin/rails g stepper_motor:install --uuid # Pass "uuid" if you are using UUID for your primary and foreign keys
|
83
36
|
$ bin/rails db:migrate
|
84
37
|
|
85
|
-
##
|
86
|
-
|
87
|
-
Define a workflow and launch your user into it:
|
88
|
-
|
89
|
-
```ruby
|
90
|
-
class SignupJourney < StepperMotor::Journey
|
91
|
-
step :after_signup do
|
92
|
-
WelcomeMailer.welcome_email(hero).deliver_later
|
93
|
-
end
|
94
|
-
|
95
|
-
step :remind_of_tasks, wait: 2.days do
|
96
|
-
ServiceUpdateMailer.two_days_spent_email(hero).deliver_later
|
97
|
-
end
|
98
|
-
|
99
|
-
step :onboarding_complete_, wait: 15.days do
|
100
|
-
OnboardingCompleteMailer.onboarding_complete_email(hero).deliver_later
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
class SignupController
|
105
|
-
def create
|
106
|
-
# ...your other business actions
|
107
|
-
SignupJourney.create!(hero: current_user)
|
108
|
-
redirect_to user_root_path(current_user)
|
109
|
-
end
|
110
|
-
end
|
111
|
-
```
|
112
|
-
|
113
|
-
## A few sample journeys: single step with repeats
|
114
|
-
|
115
|
-
Let's examine a simple single-step journey. Imagine you have a user that is about to churn, and you want to keep sending them drip emails until they churn in the hope that they will reconvert. The Journey will likely look like this:
|
116
|
-
|
117
|
-
```ruby
|
118
|
-
class ChurnPreventionJourney < StepperMotor::Journey
|
119
|
-
step do
|
120
|
-
cancel! if hero.subscription_lapses_at > 120.days.from_now
|
121
|
-
|
122
|
-
time_remaining_until_expiry_ = hero.subscription_lapses_at - Time.current
|
123
|
-
if time_remaining_until_expiry > 1.days
|
124
|
-
ResubscribeReminderMailer.extend_subscription_reminder(hero).deliver_later
|
125
|
-
send_next_reminder_after = (time_remaining_until_expiry / 2).in_days.floor
|
126
|
-
reattempt!(wait: send_next_reminder_after.days)
|
127
|
-
else
|
128
|
-
# If the user has churned - let the journey finish, as there is nothing to do
|
129
|
-
SadToSeeYouGoMailer.farewell(hero).deliver_later
|
130
|
-
end
|
131
|
-
end
|
132
|
-
end
|
133
|
-
|
134
|
-
ChurnPreventionJourney.create(hero: user)
|
135
|
-
```
|
136
|
-
|
137
|
-
In this case we have just one `step` which is going to be repeated. When we decide to repeat a step (if the user still has time to reconnect with the business), we postpone its execution by a certain amount of time - in this case, half the days remaining on the user's subscription. If a user rescubscribes, we `cancel!` the only step of the `Journey`, after which it gets marked `finished` in the database.
|
138
|
-
|
139
|
-
## More sample journeys: email drip campaign
|
140
|
-
|
141
|
-
As our second example, let's check out a drip campaign which inceitivises a user with bonuses as their account nears termination.
|
142
|
-
|
143
|
-
```ruby
|
144
|
-
class ReengagementJourney < StepperMotor::Journey
|
145
|
-
step :first do
|
146
|
-
cancel! if reengaged?
|
147
|
-
hero.bonus_programs.create!(type: BonusProgram::REENGAGEMENT)
|
148
|
-
hero.push_anayltics_event!(event: "reengagement_touchpoint", properties: {step: 1})
|
149
|
-
end
|
150
|
-
|
151
|
-
step :second, wait: 14.days do
|
152
|
-
cancel! if reengaged?
|
153
|
-
hero.bonus_programs.create!(type: BonusProgram::DISCOUNT)
|
154
|
-
hero.push_anayltics_event!(event: "reengagement_touchpoint", properties: {step: 2})
|
155
|
-
end
|
156
|
-
|
157
|
-
step :third, wait: 7.days do
|
158
|
-
cancel! if reengaged?
|
159
|
-
hero.bonus_programs.create!(type: BonusProgram::DOUBLE_DISCOUNT)
|
160
|
-
hero.push_anayltics_event!(event: "reengagement_touchpoint", properties: {step: 3})
|
161
|
-
end
|
162
|
-
|
163
|
-
step :final, wait: 3.days do
|
164
|
-
cancel! if reengaged?
|
165
|
-
hero.close_account!
|
166
|
-
hero.push_anayltics_event!(event: "reengagement_touchpoint", properties: {step: 4})
|
167
|
-
end
|
168
|
-
|
169
|
-
def reengaged?
|
170
|
-
# If the user purchased anything after this journey started,
|
171
|
-
# consider them "re-engaged"
|
172
|
-
hero.purchases.where("created_at > ?", created_at).any?
|
173
|
-
end
|
174
|
-
end
|
175
|
-
```
|
176
|
-
|
177
|
-
In this instance, we split our workflow in a number of steps - 4 in total. After the first step (`:first`) we wait for 14 days before executing the next one. 7 days later - we run another one. We end with closing the user's account. If the user has reengaged at any step, we mark the `Journey` as `canceled`.
|
178
|
-
|
179
|
-
## More sample journeys: archiving and deleting user data
|
180
|
-
|
181
|
-
Imagine a user on your platform has requested their account to be deleted. Usually you do some archiving before deletion, to preserve some data that can be useful in aggregate - just scrubbing the PII. You also change the user information so that the user does not show up in the normal application flows anymore.
|
182
|
-
|
183
|
-
```ruby
|
184
|
-
class AccountErasureJourney < StepperMotor::Journey
|
185
|
-
step :deactivate_user do
|
186
|
-
hero.deactivated!
|
187
|
-
end
|
188
|
-
|
189
|
-
step :remove_authentication_tokens do
|
190
|
-
hero.sessions.destroy_all
|
191
|
-
hero.authentication_tokens.destroy_all
|
192
|
-
end
|
193
|
-
|
194
|
-
step :archive_pseudonymized_data do
|
195
|
-
DatapointArchive.create(name> "user-#{hero.id}-datapoints.gz") do |io|
|
196
|
-
CSV(io) do |csv|
|
197
|
-
csv << hero.datapoints.first.attributes.keys
|
198
|
-
hero.datapoints.each do |datapoint|
|
199
|
-
csv << Pseudonymizer.scrub(datapoint.attributes.values)
|
200
|
-
end
|
201
|
-
end
|
202
|
-
end
|
203
|
-
end
|
204
|
-
|
205
|
-
step :delete_data do
|
206
|
-
hero.datapoints.in_batches.destroy_all
|
207
|
-
end
|
208
|
-
|
209
|
-
step :send_deletion_email do
|
210
|
-
AccountErasureCompleteMailer.erasure_complete(hero).deliver_later
|
211
|
-
end
|
212
|
-
end
|
213
|
-
```
|
214
|
-
|
215
|
-
While this is seemingly overkill to have steps defined for this type of workflow, the basic premise of a `Journey` still offers you substantial benefits. For example, you never want to enter `delete_data` before `archive_pseudonymized_data` has completed. Same with the `send_deletion_email` - you do not want to notify the user berore their data is actually gone. Neither do you want there to ever be more than 1 process executing any of those steps.
|
216
|
-
|
217
|
-
## More sample journeys: performing an outgoing payment
|
38
|
+
## 🚧 stepper_motor is undergoing active development
|
218
39
|
|
219
|
-
|
40
|
+
For versions 0.1.x stepper_motor is going to undergo active development, with infrequent - but possible - API changes and database schema changes. Here's what it means for you:
|
220
41
|
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
ik = hero.idempotency_key # The `hero` in this case is a Payment, not the User
|
225
|
-
result = PaymentProvider.transfer!(
|
226
|
-
from_account: hero.sender.bank_account_details,
|
227
|
-
to_account: hero.recipient.bank_account_details,
|
228
|
-
amount: hero.amount,
|
229
|
-
idempotency_key: ik
|
230
|
-
)
|
231
|
-
if result.intermittent_error?
|
232
|
-
reattempt!(wait: 5.seconds)
|
233
|
-
elsif result.invalid_request?
|
234
|
-
hero.failed!
|
235
|
-
cancel!
|
236
|
-
else
|
237
|
-
hero.processing!
|
238
|
-
# and then do nothing and proceed to the next step
|
239
|
-
end
|
240
|
-
end
|
241
|
-
|
242
|
-
step :confirm_payment do
|
243
|
-
ik = hero.idempotency_key # The `hero` in this case is a Payment, not the User
|
244
|
-
payment_details = PaymentProvider.details(idempotency_key: ik)
|
245
|
-
case payment_details.state
|
246
|
-
when :complete
|
247
|
-
hero.complete!
|
248
|
-
PaymentSentNotification.notify_sender_of_success(hero.sender).deliver_later
|
249
|
-
when :failed
|
250
|
-
hero.failed!
|
251
|
-
PaymentSentNotification.notify_sender_of_failure(hero.sender).deliver_later
|
252
|
-
else
|
253
|
-
logger.info {"Payment #{hero} still confirming" }
|
254
|
-
reattempt!(wait: 30.seconds) if payment_details.state == :processing
|
255
|
-
end
|
256
|
-
end
|
257
|
-
end
|
258
|
-
```
|
259
|
-
|
260
|
-
Here, we first initiate a payment using an idempotency key, and then poll for its completion or failure repeatedly. When a payment fails or succeeds, we notify the sender and finish the `Journey`. Note that this `Journey` is of a _payment,_ not of the user. A user may have multiple Payments in flight, each with their own `Journey` being tracket transactionally and correctly.
|
261
|
-
|
262
|
-
## Transactional semantics
|
263
|
-
|
264
|
-
Getting the transactional semantics _right_ with a system like StepperMotor is crucial. We strike a decent balance between reliability/durability and performance, namely:
|
265
|
-
|
266
|
-
* The initial "checkout" of a `Journey` for performing a step is lock-guarded
|
267
|
-
* Inside the lock guard the `state` of the `Journey` gets set to `performing` - you can see that a journey is currently being performed, and no other processes will evern checkout that same `Journey`
|
268
|
-
* The transaction is only applied at the start of the step, _outside_ of that step's block. This means that you can perform long-running operations in your steps, as long as they are idempotent - and manage transactions inside of the steps.
|
269
|
-
|
270
|
-
We chose to make StepperMotor "transactionless" inside the steps because the operations and side effects we usually care about would be long-running and performing HTTP or RPC requests. Had the step been wrapped with a transaction, the transaction could become very long - creating a potential for a fairly large rollback in case the step fails.
|
271
|
-
|
272
|
-
Another reason why we avoid forced transactions is that if, for whatever reason, you need multiple idempotent actions _inside_ of a step the outer transaction would not permit you to have those. We prefer leaving that flexibility to the end application.
|
42
|
+
* When you update the gem you must run `bin/rails g stepper_motor:install --skip` to add any migrations you may need
|
43
|
+
* You have to ensure the application running a new version has the DB schema already updated. If your deployment does not allow for automatic
|
44
|
+
sequential migrate-then-deploy, deploy the migrations first.
|
273
45
|
|
274
|
-
|
275
|
-
|
276
|
-
Right now, StepperMotor does not provide any specific tools for saving side-effects or inputs of a step or of the entire `Journey` except for the related `hero` record. The reason for that is that side effects can take many shapes. A side effect may be a file output to S3, a record saved into your database, a file on the filesystem, or a blob of JSON carried around. The way this data has to be persisted can also vary. For the moment, we don't see a good _generalized_ way to persist those side effects aside of the factual outputs. So:
|
277
|
-
|
278
|
-
* A record of the fact that a step has been performed to completion is sufficient to not re-enter that step
|
279
|
-
* If you need repeatable, but idempotent steps - idempotency is on you
|
280
|
-
|
281
|
-
## Unique Journeys
|
282
|
-
|
283
|
-
By default, StepperMotor will only allow you to have one active `Journey` per journey type for any given specific `hero`. This will fail, either with a uniqueness constraint violation or a validation error:
|
284
|
-
|
285
|
-
```ruby
|
286
|
-
SomeJourney.create!(hero: user)
|
287
|
-
SomeJourney.create!(hero: user)
|
288
|
-
```
|
289
|
-
|
290
|
-
Once a `Journey` becomes `canceled` or `finished`, another `Journey` of the same class can be created again for the same `hero`. If you need to create multiple `Journeys` of the same class for the same `hero`, pass the `allow_multiple` attribute set to `true`. This value gets persisted and affects the inclusion of the `Journey` into a partial index that enforces uniqueness:
|
291
|
-
|
292
|
-
```ruby
|
293
|
-
SomeJourney.create!(hero: user, allow_multiple: true)
|
294
|
-
SomeJourney.create!(hero: user, allow_multiple: true)
|
295
|
-
```
|
296
|
-
|
297
|
-
## Querying for Journeys already created
|
298
|
-
|
299
|
-
Frequently, you will encounter the need to select `heroes` to create `Journeys` for. You will likely want to create `Journeys` only for those `heroes` who do not have these `Journeys` yet. You can use a shortcut to generate you the SQL query to use in a `WHERE NOT EXISTS` SQL clause. Usually, your query will look something like this:
|
300
|
-
|
301
|
-
```sql
|
302
|
-
SELECT users.* FROM users WHERE NOT EXISTS (SELECT 1 FROM stepper_motor_journeys WHERE type = 'YourJourney' AND hero_id = users.id)
|
303
|
-
```
|
304
|
-
|
305
|
-
To make this simpler, we offer a special helper method:
|
306
|
-
|
307
|
-
```ruby
|
308
|
-
YourJourney.presence_sql_for(User) # => SELECT 1 FROM stepper_motor_journeys WHERE type = 'YourJourney' AND hero_id = users.id
|
309
|
-
```
|
310
|
-
|
311
|
-
## What to pick as the hero
|
312
|
-
|
313
|
-
If your use case requires complex associations, you may want to make your `hero` a record representing the business process that the `Journey` tracks, instead of making the "actor" (say, an `Account`) the hero. This will allow for better granularity and better-looking code that will be easier to understand.
|
314
|
-
|
315
|
-
So instead of doing this:
|
316
|
-
|
317
|
-
```ruby
|
318
|
-
class PurchaseJourney < StepperMotor::Journey
|
319
|
-
step :start_checkout do
|
320
|
-
hero.purchases.create!(sku: ...)
|
321
|
-
end
|
322
|
-
end
|
323
|
-
|
324
|
-
PurchaseJourney.create!(hero: user, allow_multiple: true)
|
325
|
-
```
|
326
|
-
|
327
|
-
try this:
|
328
|
-
|
329
|
-
```ruby
|
330
|
-
class PurchaseJourney < StepperMotor::Journey
|
331
|
-
step :start_checkout do
|
332
|
-
hero.checkout_started!
|
333
|
-
end
|
334
|
-
end
|
335
|
-
|
336
|
-
purchase = user.purchases.create!(sku: ...)
|
337
|
-
PurchaseJourney.create!(hero: purchase)
|
338
|
-
```
|
339
|
-
|
340
|
-
## Forward-scheduling or in-time scheduling
|
341
|
-
|
342
|
-
There are two known approaches for scheduling jobs far into the future. One approach is "in-time scheduling" - regularly run a _scheduling task_ which performs the steps that are up for execution. The code for such process would look roughly looks like this:
|
343
|
-
|
344
|
-
```ruby
|
345
|
-
Journey.where("state = 'ready' AND next_step_to_be_performed_at <= NOW()").find_each(&:perform_next_step!)
|
346
|
-
````
|
347
|
-
|
348
|
-
This scheduling task needs to be run with a high-enough frequency which matches your scheduling patterns.
|
349
|
-
|
350
|
-
Another is "forward-scheduling" - when it is known that a step of a journey will have to be performed at a certain point in time, enqueue a job which is going to perform the step:
|
351
|
-
|
352
|
-
```ruby
|
353
|
-
PerformStepJob.set(wait: journey.next_step_to_be_performed_at).perform_later(journey)
|
354
|
-
```
|
355
|
-
|
356
|
-
This creates a large number of jobs on your queue, but will be easier to manage. StepperMotor supports both approaches, and you can configure the one you like using the configuration:
|
357
|
-
|
358
|
-
```ruby
|
359
|
-
StepperMotor.configure do |c|
|
360
|
-
# Use jobs per journey step and enqueue them early
|
361
|
-
c.scheduler = StepperMotor::ForwardScheduler.new
|
362
|
-
end
|
363
|
-
```
|
364
|
-
|
365
|
-
or, for cyclic scheduling (less jobs on the queue, but you need a decent scheduler for your background jobs to be present:
|
366
|
-
|
367
|
-
```ruby
|
368
|
-
StepperMotor.configure do |c|
|
369
|
-
# Check for jobs to be created every 5 minutes
|
370
|
-
c.scheduler = StepperMotor::CyclicScheduler.new((cycle_duration: 5.minutes)
|
371
|
-
end
|
372
|
-
```
|
373
|
-
|
374
|
-
If you use in-time scheduling you will need to add the `StepperMotor::ScheduleLoopJob` to your cron jobs, and perform it frequently enough. Note that having just the granularity of your cron jobs (minutes) may not be enough as reattempts of the steps may get scheduled with a smaller delay - of a few seconds, for instance.
|
375
|
-
|
376
|
-
## Naming steps
|
377
|
-
|
378
|
-
stepper_motor will name steps for you. However, using named steps is useful because you then can insert steps between existing ones, and have your `Journey` correctly identify the right step. Steps are performed in the order they are defined. Imagine you start with this step sequence:
|
379
|
-
|
380
|
-
```ruby
|
381
|
-
step :one do
|
382
|
-
# perform some action
|
383
|
-
end
|
384
|
-
|
385
|
-
step :two do
|
386
|
-
# perform some other action
|
387
|
-
end
|
388
|
-
```
|
389
|
-
|
390
|
-
You have a `Journey` which is about to start step `one`. When the step gets performed, StepperMotor will do a lookup to find _the next step in order of definition._ In this case the step will be step `two`, so the name of that step will be saved with the `Journey`. Imagine you then edit the code to add an extra step between those:
|
391
|
-
|
392
|
-
```ruby
|
393
|
-
step :one do
|
394
|
-
# perform some action
|
395
|
-
end
|
396
|
-
|
397
|
-
step :one_bis_ do
|
398
|
-
# some compliance action
|
399
|
-
end
|
400
|
-
|
401
|
-
step :two do
|
402
|
-
# perform some other action
|
403
|
-
end
|
404
|
-
```
|
405
|
-
|
406
|
-
Your existing `Journey` is already primed to perform step `two`. However, a `Journey` which is about to perform step `one` will now set `one_bis` as the next step to perform. This allows limited reordering and editing of `Journey` definitions after they have already begun.
|
407
|
-
|
408
|
-
So, rules of thumb:
|
409
|
-
|
410
|
-
* When steps are recalled to be performed, they get recalled _by name._
|
411
|
-
* When preparing for the next step, _the next step from the current in order of definition_ is going to be used.
|
46
|
+
Starting with versions 0.2.x and up, stepper_motor will make the best possible effort to allow operation without having applied the recent migrations.
|
412
47
|
|
413
48
|
## Development
|
414
49
|
|
@@ -416,6 +51,10 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
|
|
416
51
|
|
417
52
|
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
418
53
|
|
54
|
+
## Is it any good?
|
55
|
+
|
56
|
+
[Yes.](https://news.ycombinator.com/item?id=3067434)
|
57
|
+
|
419
58
|
## Contributing
|
420
59
|
|
421
60
|
Bug reports and pull requests are welcome on GitHub at https://github.com/stepper-motor/stepper_motor.
|
data/Rakefile
CHANGED
@@ -1,13 +1,31 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "bundler/gem_tasks"
|
4
|
-
require "
|
4
|
+
require "rake/testtask"
|
5
5
|
require "standard/rake"
|
6
|
+
require "yard"
|
7
|
+
|
8
|
+
YARD::Rake::YardocTask.new(:doc)
|
9
|
+
|
10
|
+
Rake::TestTask.new(:test) do |t|
|
11
|
+
t.libs << "test"
|
12
|
+
t.libs << "lib"
|
13
|
+
t.test_files = FileList["test/**/*_test.rb"]
|
14
|
+
t.warning = false # To avoid any warnings from dependencies
|
15
|
+
end
|
6
16
|
|
7
17
|
task :format do
|
8
18
|
`bundle exec standardrb --fix`
|
9
19
|
`bundle exec magic_frozen_string_literal .`
|
10
20
|
end
|
11
21
|
|
12
|
-
|
13
|
-
|
22
|
+
task :generate_typedefs do
|
23
|
+
`bundle exec sord rbi/stepper_motor.rbi`
|
24
|
+
`bundle exec sord sig/stepper_motor.rbs`
|
25
|
+
end
|
26
|
+
|
27
|
+
task default: [:test, :standard, :generate_typedefs]
|
28
|
+
|
29
|
+
# When building the gem, generate typedefs beforehand,
|
30
|
+
# so that they get included
|
31
|
+
Rake::Task["build"].enhance(["generate_typedefs"])
|
data/bin/test
ADDED
@@ -8,10 +8,15 @@ module StepperMotor
|
|
8
8
|
# initializer and the migration that creates tables.
|
9
9
|
# Run it with `bin/rails g stepper_motor:install` in your console.
|
10
10
|
class InstallGenerator < Rails::Generators::Base
|
11
|
+
UUID_MESSAGE = <<~MSG
|
12
|
+
If set, uuid type will be used for hero_id. Use this
|
13
|
+
if most of your models use UUD as primary key"
|
14
|
+
MSG
|
15
|
+
|
11
16
|
include ActiveRecord::Generators::Migration
|
12
17
|
|
13
18
|
source_paths << File.join(File.dirname(__FILE__, 2))
|
14
|
-
class_option :uuid, type: :boolean, desc:
|
19
|
+
class_option :uuid, type: :boolean, desc: UUID_MESSAGE
|
15
20
|
|
16
21
|
# Generates monolithic migration file that contains all database changes.
|
17
22
|
def create_migration_file
|