pipeline 0.0.8 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +2 -1
- data/CHANGELOG +14 -0
- data/README.rdoc +31 -9
- data/Rakefile +3 -1
- data/TODO +1 -2
- data/VERSION +1 -1
- data/generators/pipeline/templates/migration.rb +3 -0
- data/lib/pipeline.rb +2 -1
- data/lib/pipeline/api_methods.rb +15 -1
- data/lib/pipeline/base.rb +204 -13
- data/lib/pipeline/core_ext/symbol_attribute.rb +16 -2
- data/lib/pipeline/core_ext/transactional_attribute.rb +18 -1
- data/lib/pipeline/errors.rb +18 -2
- data/lib/pipeline/stage/base.rb +141 -8
- data/pipeline.gemspec +13 -6
- data/spec/database_integration_helper.rb +1 -0
- data/spec/models.rb +72 -0
- data/spec/pipeline/api_methods_spec.rb +5 -4
- data/spec/pipeline/base_spec.rb +119 -80
- data/spec/pipeline/core_ext/symbol_attribute_spec.rb +1 -1
- data/spec/pipeline/core_ext/transactional_attribute_spec.rb +1 -1
- data/spec/pipeline/errors_spec.rb +1 -1
- data/spec/pipeline/stage/base_spec.rb +103 -88
- data/spec/spec_helper.rb +2 -0
- metadata +7 -4
data/.gitignore
CHANGED
data/CHANGELOG
CHANGED
@@ -1,3 +1,17 @@
|
|
1
|
+
0.0.9
|
2
|
+
=====
|
3
|
+
|
4
|
+
Bug Fix:
|
5
|
+
* Syntax error on generated migration
|
6
|
+
|
7
|
+
Features:
|
8
|
+
* Upgrading dependency to delayed_job
|
9
|
+
* Move to gemcutter
|
10
|
+
* Can be installed as a plugin or gem
|
11
|
+
* Improved Documentation
|
12
|
+
* Callbacks before and after stage/pipeline execution
|
13
|
+
* Association to an external entity through external_id (Please refer to documentation)
|
14
|
+
|
1
15
|
0.0.8
|
2
16
|
=====
|
3
17
|
|
data/README.rdoc
CHANGED
@@ -4,6 +4,10 @@
|
|
4
4
|
|
5
5
|
Pipeline is a Rails plugin/gem to run asynchronous processes in a configurable pipeline.
|
6
6
|
|
7
|
+
== Documentation
|
8
|
+
|
9
|
+
http://rdoc.info/projects/dtsato/pipeline
|
10
|
+
|
7
11
|
== Features
|
8
12
|
|
9
13
|
* Execution of sequential user-defined stages in an asynchronous pipeline
|
@@ -12,36 +16,54 @@ Pipeline is a Rails plugin/gem to run asynchronous processes in a configurable p
|
|
12
16
|
* Irrecoverable errors fail the entire pipeline
|
13
17
|
* Recoverable errors are automatically retried (using dj's exponential retry strategy)
|
14
18
|
* Recoverable errors that require user input pause the pipeline for further retry
|
15
|
-
* Cancelling of a paused pipeline
|
19
|
+
* Cancelling/Resuming of a paused pipeline
|
20
|
+
* Callbacks before and after executing stages and pipeline
|
21
|
+
|
22
|
+
== Installation and Use
|
16
23
|
|
17
|
-
|
18
|
-
* Rails' tests/specs will only work if delayed_job is installed as a gem, rather than a plugin.
|
24
|
+
+pipeline+ can be installed as either a RubyGem (recommended) or as a plugin.
|
19
25
|
|
20
|
-
|
26
|
+
=== Gem
|
21
27
|
|
22
|
-
|
28
|
+
To install +pipeline+ as a RubyGem, add the following lines to your
|
29
|
+
<tt>config/environment.rb</tt> file:
|
23
30
|
|
24
31
|
config.gem "pipeline"
|
32
|
+
config.gem "delayed_job"
|
25
33
|
|
26
|
-
|
34
|
+
And execute:
|
27
35
|
|
28
36
|
rake gems:install
|
29
37
|
rake gems:unpack # Optional, if you want to vendor the gem
|
38
|
+
|
39
|
+
=== Plugin
|
40
|
+
|
41
|
+
To install it as a plugin, run:
|
42
|
+
|
43
|
+
script/plugin install git://github.com/dtsato/pipeline.git
|
44
|
+
|
45
|
+
=== Generating the required tables
|
46
|
+
|
47
|
+
In order to persist your pipelines and stages, execute the following:
|
48
|
+
|
30
49
|
script/generate pipeline # To generate the migration scripts that will store pipelines
|
50
|
+
script/generate delayed_job # To generate the migration scripts for delayed_job
|
31
51
|
rake db:migrate
|
32
52
|
|
53
|
+
=== Starting your workers
|
54
|
+
|
33
55
|
You will also need to run your Delayed Job workers that will process the pipeline jobs in the background:
|
34
56
|
|
35
57
|
rake jobs:work
|
36
58
|
|
37
59
|
== Dependencies
|
38
60
|
|
39
|
-
* Rails
|
40
|
-
* Delayed job (http://github.com/collectiveidea/delayed_job
|
61
|
+
* Rails (ActiveRecord)
|
62
|
+
* Delayed job (http://github.com/collectiveidea/delayed_job)
|
41
63
|
|
42
64
|
== Usage
|
43
65
|
|
44
|
-
Check
|
66
|
+
Check +examples+ for more examples (including error-recovery and cancelling)
|
45
67
|
|
46
68
|
class Step1 < Pipeline::Stage::Base
|
47
69
|
def perform
|
data/Rakefile
CHANGED
@@ -18,10 +18,12 @@ begin
|
|
18
18
|
gem.test_files = Dir['spec/**/*'] + Dir['spec/*']
|
19
19
|
|
20
20
|
gem.add_dependency('activerecord', '>= 2.0')
|
21
|
-
gem.add_dependency('
|
21
|
+
gem.add_dependency('delayed_job', '>= 1.8.0')
|
22
22
|
|
23
23
|
gem.rubyforge_project = "pipeline"
|
24
24
|
end
|
25
|
+
|
26
|
+
Jeweler::GemcutterTasks.new
|
25
27
|
|
26
28
|
rescue LoadError
|
27
29
|
puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
|
data/TODO
CHANGED
@@ -1,7 +1,6 @@
|
|
1
|
-
* Allow delayed_job to be used as a plugin, as well as a gem
|
2
1
|
* Add examples of view display
|
3
2
|
* view helpers?
|
4
|
-
*
|
3
|
+
* Chaining stages with string (to avoid having to load stages before pipeline)
|
5
4
|
* Adapter for other persistence frameworks (ActiveModel?)
|
6
5
|
* Adapter for other background processing mechanisms (currently relying on dj's auto-retry)
|
7
6
|
* Max attempts on pipeline (or stage)
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.9
|
@@ -4,6 +4,8 @@ class CreatePipelineInstancesAndStages < ActiveRecord::Migration
|
|
4
4
|
t.string :type # For single table inheritance
|
5
5
|
t.string :status # Current status of the pipeline
|
6
6
|
t.integer :attempts, :default => 0 # Number of times this pipeline was executed
|
7
|
+
t.references :external # External object, to which this pipeline
|
8
|
+
# is associated (user-defined)
|
7
9
|
|
8
10
|
t.timestamps
|
9
11
|
end
|
@@ -17,6 +19,7 @@ class CreatePipelineInstancesAndStages < ActiveRecord::Migration
|
|
17
19
|
t.integer :attempts, :default => 0 # Number of times this stage was executed
|
18
20
|
|
19
21
|
t.timestamps
|
22
|
+
end
|
20
23
|
end
|
21
24
|
|
22
25
|
def self.down
|
data/lib/pipeline.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
autoload :ActiveRecord, '
|
1
|
+
autoload :ActiveRecord, 'active_record'
|
2
2
|
autoload :Delayed, 'delayed_job'
|
3
3
|
|
4
4
|
$: << File.dirname(__FILE__)
|
@@ -9,6 +9,7 @@ require 'pipeline/base'
|
|
9
9
|
require 'pipeline/errors'
|
10
10
|
require 'pipeline/stage/base'
|
11
11
|
|
12
|
+
# Please refer to Pipeline::Base and Pipeline::Stage::Base for detailed documentation
|
12
13
|
module Pipeline
|
13
14
|
extend(ApiMethods)
|
14
15
|
end
|
data/lib/pipeline/api_methods.rb
CHANGED
@@ -1,5 +1,11 @@
|
|
1
1
|
module Pipeline
|
2
|
+
# This is the external API for Pipeline. Its methods should be called by client
|
3
|
+
# code wanting to manipulate/execute pipelines.
|
2
4
|
module ApiMethods
|
5
|
+
|
6
|
+
# Used to enqueue a pipeline execution. Raises InvalidPipelineError if the passed
|
7
|
+
# in argument is not a subclass of Pipeline::Base. The pipeline will be saved (if
|
8
|
+
# not already) and its <tt>id</tt> will be returned.
|
3
9
|
def start(pipeline)
|
4
10
|
raise InvalidPipelineError.new("Invalid pipeline") unless pipeline.is_a?(Pipeline::Base)
|
5
11
|
pipeline.save! if pipeline.new_record?
|
@@ -7,14 +13,22 @@ module Pipeline
|
|
7
13
|
pipeline.id
|
8
14
|
end
|
9
15
|
|
16
|
+
# Enqueues execution of a paused pipeline for retrying. Raises InvalidPipelineError
|
17
|
+
# if a pipeline can not be found with the provided <tt>id</tt>. Raises
|
18
|
+
# InvalidStatusError if pipeline is in an invalid state for resuming (e.g. already
|
19
|
+
# cancelled, or completed)
|
10
20
|
def resume(id)
|
11
21
|
pipeline = Base.find(id)
|
12
|
-
|
22
|
+
pipeline.resume
|
13
23
|
Delayed::Job.enqueue(pipeline)
|
14
24
|
rescue ActiveRecord::RecordNotFound
|
15
25
|
raise InvalidPipelineError.new("Invalid pipeline")
|
16
26
|
end
|
17
27
|
|
28
|
+
# Cancels execution of a paused pipeline. Raises InvalidPipelineError if a pipeline
|
29
|
+
# can not be found with the provided <tt>id</tt>. Raises InvalidStatusError if
|
30
|
+
# pipeline is in an invalid state for cancelling (e.g. already cancelled, or
|
31
|
+
# completed)
|
18
32
|
def cancel(id)
|
19
33
|
pipeline = Base.find(id)
|
20
34
|
pipeline.cancel
|
data/lib/pipeline/base.rb
CHANGED
@@ -1,16 +1,163 @@
|
|
1
1
|
module Pipeline
|
2
|
+
# == Pipeline Stages
|
3
|
+
#
|
4
|
+
# Each pipeline is composed of sequential stages (see Pipeline::Stage::Base).
|
5
|
+
# The stages that will be executed are defined as follows:
|
6
|
+
#
|
7
|
+
# class PrepareIngredients < Pipeline::Stage::Base
|
8
|
+
# def run
|
9
|
+
# puts "Slicing..."
|
10
|
+
# end
|
11
|
+
# end
|
12
|
+
#
|
13
|
+
# class Cook < Pipeline::Stage::Base
|
14
|
+
# def run
|
15
|
+
# puts "Cooking..."
|
16
|
+
# end
|
17
|
+
# end
|
18
|
+
#
|
19
|
+
# class MakeDinnerPipeline < Pipeline::Base
|
20
|
+
# define_stages PrepareIngredients >> Cook
|
21
|
+
# end
|
22
|
+
#
|
23
|
+
# When this pipeline executes, it will run each stage sequentially, and the output
|
24
|
+
# would be:
|
25
|
+
# Slicing...
|
26
|
+
# Cooking...
|
27
|
+
#
|
28
|
+
# A pipeline can get access to its stages through the <tt>stages</tt> association.
|
29
|
+
#
|
30
|
+
# == Error Handling
|
31
|
+
#
|
32
|
+
# There are 3 types of errors that a failed stage can specifically raise:
|
33
|
+
#
|
34
|
+
# * <b>Recoverable (requires user action)</b>: If a stage raises RecoverableError with
|
35
|
+
# <tt>input_required? == true</tt>, the pipeline gets :paused and can be
|
36
|
+
# resumed or cancelled by calling #resume and #cancel, respectively.
|
37
|
+
#
|
38
|
+
# * <b>Recoverable (can be automatically retried)</b>: If a stage raises
|
39
|
+
# RecoverableError with <tt>input_required? == false</tt>, the pipeline goes into
|
40
|
+
# :retry state and will be automatically retried. This is currently achieved by
|
41
|
+
# +delayed_job+'s retry mechanism. Please refer to
|
42
|
+
# http://github.com/collectiveidea/delayed_job for information about how to
|
43
|
+
# configure the maximum number of retry attempts.
|
44
|
+
#
|
45
|
+
# * <b>Irrecoverable</b>: If a stage fails with an IrrecoverableError, the pipeline
|
46
|
+
# gets :failed and therefore cannot be resumed or restarted.
|
47
|
+
#
|
48
|
+
# If a stage fails with any other type of error, you can choose the default behaviour
|
49
|
+
# for what happens to the pipeline. By default, the pipeline will pause, so it can be
|
50
|
+
# later resumed. This can be overriden by calling +default_failure_mode+ like:
|
51
|
+
#
|
52
|
+
# class SamplePipeline < Pipeline::Base
|
53
|
+
# self.default_failure_mode = :cancel
|
54
|
+
# end
|
55
|
+
#
|
56
|
+
# You can always go back to the default mode by calling:
|
57
|
+
# self.default_failure_mode = :pause
|
58
|
+
#
|
59
|
+
# == State Transitions
|
60
|
+
#
|
61
|
+
# The following diagram represents the state transitions a pipeline instance can
|
62
|
+
# go through during its life-cycle:
|
63
|
+
#
|
64
|
+
# :not_started ---> :in_progress ---> :completed / :failed
|
65
|
+
# ^ |
|
66
|
+
# | v
|
67
|
+
# :paused / :retry
|
68
|
+
#
|
69
|
+
# [:not_started] The pipeline was instantiated but not started yet.
|
70
|
+
# [:in_progress] After started or resumed, the pipeline remains on this state while
|
71
|
+
# the stages are running.
|
72
|
+
# [:paused] If a stage fails with a recoverable error that requires user action,
|
73
|
+
# the pipeline gets paused.
|
74
|
+
# [:retry] If a stage fails with a recoverable error that can be automatically
|
75
|
+
# retried, the pipeline goes into this stage.
|
76
|
+
# [:completed] After successfully running all stages, the pipeline is completed.
|
77
|
+
# [:failed] If a stage fails with an unrecoverable error, or if the pipeline is
|
78
|
+
# cancelled, it goes into this stage.
|
79
|
+
#
|
80
|
+
# == Referencing External Objects
|
81
|
+
#
|
82
|
+
# The execution of a pipeline will usually be associated to an external entity
|
83
|
+
# (e.g. a +User+ if the stages represent an internal user registration process, or a
|
84
|
+
# +Recipe+ in the examples of this page). To be able to reference the associated object
|
85
|
+
# from the stages, Pipeline::Base has an attribute <tt>external_id</tt> that can be
|
86
|
+
# used on a custom association to any external entity. Example:
|
87
|
+
#
|
88
|
+
# class MakeDinnerPipeline < Pipeline::Base
|
89
|
+
# define_stages PrepareIngredients >> Cook
|
90
|
+
# belongs_to :recipe, :foreign_key => 'external_id'
|
91
|
+
# end
|
92
|
+
#
|
93
|
+
# A Stage can reference this object as such:
|
94
|
+
#
|
95
|
+
# class Cook < Pipeline::Stage::Base
|
96
|
+
# def run
|
97
|
+
# puts "Cooking a delicious #{pipeline.recipe.name}"
|
98
|
+
# end
|
99
|
+
# end
|
100
|
+
#
|
101
|
+
# == Callbacks
|
102
|
+
#
|
103
|
+
# You can define custom callbacks to be called before (+before_pipeline+) and after
|
104
|
+
# (+after_pipeline+) executing a pipeline. Example:
|
105
|
+
#
|
106
|
+
# class PrepareIngredients < Pipeline::Stage::Base
|
107
|
+
# def run
|
108
|
+
# puts "Slicing..."
|
109
|
+
# end
|
110
|
+
# end
|
111
|
+
#
|
112
|
+
# class Cook < Pipeline::Stage::Base
|
113
|
+
# def run
|
114
|
+
# puts "Cooking..."
|
115
|
+
# end
|
116
|
+
# end
|
117
|
+
#
|
118
|
+
# class MakeDinnerPipeline < Pipeline::Base
|
119
|
+
# define_stages PrepareIngredients >> Cook
|
120
|
+
#
|
121
|
+
# before_pipeline :wash_hands
|
122
|
+
# after_pipeline :serve_dinner
|
123
|
+
#
|
124
|
+
# private
|
125
|
+
# def wash_hands
|
126
|
+
# puts "Washing hands before we start..."
|
127
|
+
# end
|
128
|
+
#
|
129
|
+
# def serve_dinner
|
130
|
+
# puts "bon appetit!"
|
131
|
+
# end
|
132
|
+
# end
|
133
|
+
#
|
134
|
+
# Pipeline.start(MakeDinnerPipeline.new)
|
135
|
+
#
|
136
|
+
# Outputs:
|
137
|
+
# Washing hands before we start...
|
138
|
+
# Slicing...
|
139
|
+
# Cooking...
|
140
|
+
# bon appetit!
|
141
|
+
#
|
142
|
+
# Callbacks can be defined as a symbol that calls a private/protected method (like the
|
143
|
+
# example above), as an inline block, or as a +Callback+ object, as a regular
|
144
|
+
# +ActiveRecord+ callback.
|
2
145
|
class Base < ActiveRecord::Base
|
3
146
|
set_table_name :pipeline_instances
|
4
147
|
|
5
|
-
# :not_started ---> :in_progress ---> :completed
|
6
|
-
# ^ |
|
148
|
+
# :not_started ---> :in_progress ---> :completed / :failed
|
149
|
+
# ^ |
|
7
150
|
# | v
|
8
151
|
# :paused / :retry
|
9
152
|
symbol_attr :status
|
10
153
|
transactional_attr :status
|
11
154
|
private :status=
|
12
155
|
|
13
|
-
|
156
|
+
# Allows access to the associated stages
|
157
|
+
has_many :stages,
|
158
|
+
:class_name => 'Pipeline::Stage::Base',
|
159
|
+
:foreign_key => 'pipeline_instance_id',
|
160
|
+
:dependent => :destroy
|
14
161
|
|
15
162
|
class_inheritable_accessor :defined_stages, :instance_writer => false
|
16
163
|
self.defined_stages = []
|
@@ -18,15 +165,33 @@ module Pipeline
|
|
18
165
|
class_inheritable_accessor :failure_mode, :instance_writer => false
|
19
166
|
self.failure_mode = :pause
|
20
167
|
|
168
|
+
define_callbacks :before_pipeline, :after_pipeline
|
169
|
+
|
170
|
+
# Defines the stages of this pipeline. Please refer to section
|
171
|
+
# <em>"Pipeline Stages"</em> above
|
21
172
|
def self.define_stages(stages)
|
22
173
|
self.defined_stages = stages.build_chain
|
23
174
|
end
|
24
175
|
|
176
|
+
# Sets the behaviour of this pipeline when a failure occurs. Accepted symbols are:
|
177
|
+
#
|
178
|
+
# [:pause] Pauses the pipeline on failure (default)
|
179
|
+
# [:cancel] Fails the pipeline on failure
|
25
180
|
def self.default_failure_mode=(mode)
|
26
181
|
new_mode = [:pause, :cancel].include?(mode) ? mode : :pause
|
27
182
|
self.failure_mode = new_mode
|
28
183
|
end
|
29
184
|
|
185
|
+
# Standard ActiveRecord callback to setup initial stages and status
|
186
|
+
# when a new pipeline is instantiated. If you override this callback, make
|
187
|
+
# sure to call +super+:
|
188
|
+
#
|
189
|
+
# class SamplePipeline < Pipeline::Base
|
190
|
+
# def after_initialize
|
191
|
+
# super
|
192
|
+
# self[:special_attribute] ||= "standard value"
|
193
|
+
# end
|
194
|
+
# end
|
30
195
|
def after_initialize
|
31
196
|
if new_record?
|
32
197
|
self[:status] = :not_started
|
@@ -36,42 +201,68 @@ module Pipeline
|
|
36
201
|
end
|
37
202
|
end
|
38
203
|
|
204
|
+
# Standard +delayed_job+ method called when executing this pipeline. Raises
|
205
|
+
# InvalidStatusError if pipeline is in an invalid state for execution (e.g.
|
206
|
+
# already cancelled, or completed).
|
207
|
+
#
|
208
|
+
# This method will be called by +delayed_job+
|
209
|
+
# if this object is enqueued for asynchronous execution. However, you could
|
210
|
+
# call this method and execute the pipeline synchronously, without relying on
|
211
|
+
# +delayed_job+. Auto-retry would not work in this case, though.
|
39
212
|
def perform
|
40
|
-
|
41
|
-
raise InvalidStatusError.new(status) unless ok_to_resume?
|
213
|
+
_check_valid_status
|
42
214
|
begin
|
43
215
|
_setup
|
44
216
|
stages.each do |stage|
|
45
217
|
stage.perform unless stage.completed?
|
46
218
|
end
|
47
|
-
|
219
|
+
_complete_with_status(:completed)
|
48
220
|
rescue IrrecoverableError
|
49
|
-
|
221
|
+
_complete_with_status(:failed)
|
50
222
|
rescue RecoverableError => e
|
51
223
|
if e.input_required?
|
52
|
-
|
224
|
+
_complete_with_status(:paused)
|
53
225
|
else
|
54
|
-
|
226
|
+
_complete_with_status(:retry)
|
55
227
|
raise e
|
56
228
|
end
|
57
229
|
rescue Exception
|
58
|
-
|
230
|
+
_complete_with_status(failure_mode == :cancel ? :failed : :paused)
|
59
231
|
end
|
60
232
|
end
|
61
233
|
|
234
|
+
# Attempts to cancel this pipeline. Raises InvalidStatusError if pipeline is in
|
235
|
+
# an invalid state for cancelling (e.g. already cancelled, or completed)
|
62
236
|
def cancel
|
63
|
-
|
64
|
-
|
237
|
+
_check_valid_status
|
238
|
+
_complete_with_status(:failed)
|
65
239
|
end
|
66
240
|
|
241
|
+
# Attempts to resume this pipeline. Raises InvalidStatusError if pipeline is in
|
242
|
+
# an invalid state for resuming (e.g. already cancelled, or completed)
|
243
|
+
def resume
|
244
|
+
_check_valid_status
|
245
|
+
end
|
246
|
+
|
247
|
+
private
|
67
248
|
def ok_to_resume?
|
68
249
|
[:not_started, :paused, :retry].include?(status)
|
69
250
|
end
|
70
251
|
|
71
|
-
|
252
|
+
def _check_valid_status
|
253
|
+
reload unless new_record?
|
254
|
+
raise InvalidStatusError.new(status) unless ok_to_resume?
|
255
|
+
end
|
256
|
+
|
72
257
|
def _setup
|
73
258
|
self.attempts += 1
|
74
259
|
self.status = :in_progress
|
260
|
+
run_callbacks(:before_pipeline)
|
261
|
+
end
|
262
|
+
|
263
|
+
def _complete_with_status(status)
|
264
|
+
self.status = status
|
265
|
+
run_callbacks(:after_pipeline)
|
75
266
|
end
|
76
267
|
end
|
77
268
|
end
|