dtsato-pipeline 0.0.7 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +9 -0
- data/VERSION +1 -1
- data/examples/auto_recoverable_pipeline.rb +5 -4
- data/examples/cancelling_pipeline.rb +3 -7
- data/examples/helper.rb +7 -0
- data/examples/two_step_pipeline.rb +2 -4
- data/examples/user_recoverable_pipeline.rb +4 -4
- data/lib/pipeline/api_methods.rb +1 -1
- data/lib/pipeline/base.rb +5 -4
- data/lib/pipeline/stage/base.rb +6 -3
- data/pipeline.gemspec +3 -2
- data/spec/pipeline/api_methods_spec.rb +10 -2
- data/spec/pipeline/base_spec.rb +12 -5
- data/spec/pipeline/stage/base_spec.rb +15 -0
- metadata +3 -2
data/CHANGELOG
CHANGED
@@ -1,3 +1,12 @@
|
|
1
|
+
0.0.8
|
2
|
+
=====
|
3
|
+
|
4
|
+
Bug Fix:
|
5
|
+
* Clearing message when restarting a failed stage
|
6
|
+
* Only saves the pipeline if it's a new record (client code is free to pass an already saved pipeline if needed)
|
7
|
+
* Capturing generic Exception instead of StandardError (in case of Interrupts, Timeouts, ...)
|
8
|
+
* Auto-recovering from errors was failing on InvalidStatusError. Added a new :retry status to differentiate between a paused or auto-recoverable error
|
9
|
+
|
1
10
|
0.0.7
|
2
11
|
=====
|
3
12
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.8
|
@@ -1,6 +1,4 @@
|
|
1
|
-
require File.join(File.dirname(__FILE__), '
|
2
|
-
require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
|
3
|
-
ActiveRecord::Base.logger = Logger.new(STDOUT)
|
1
|
+
require File.join(File.dirname(__FILE__), 'helper')
|
4
2
|
|
5
3
|
class Step1 < Pipeline::Stage::Base
|
6
4
|
def run
|
@@ -32,4 +30,7 @@ end
|
|
32
30
|
|
33
31
|
Pipeline.start(TwoStepPipeline.new)
|
34
32
|
|
35
|
-
Delayed::
|
33
|
+
Delayed::Job.work_off
|
34
|
+
# Waiting for second job to pass re-scheduling time limit
|
35
|
+
sleep(10)
|
36
|
+
Delayed::Job.work_off
|
@@ -1,6 +1,4 @@
|
|
1
|
-
require File.join(File.dirname(__FILE__), '
|
2
|
-
require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
|
3
|
-
ActiveRecord::Base.logger = Logger.new(STDOUT)
|
1
|
+
require File.join(File.dirname(__FILE__), 'helper')
|
4
2
|
|
5
3
|
class Step1 < Pipeline::Stage::Base
|
6
4
|
def run
|
@@ -24,9 +22,7 @@ end
|
|
24
22
|
|
25
23
|
id = Pipeline.start(TwoStepPipeline.new)
|
26
24
|
|
27
|
-
Delayed::
|
25
|
+
Delayed::Job.work_off
|
28
26
|
|
29
|
-
# CTRL-C to execute the cancelling, since we want to cancel after the stage failed, but
|
30
|
-
# Worker is blocking the process on the previous line
|
31
27
|
Pipeline.cancel(id)
|
32
|
-
|
28
|
+
puts("Pipeline is now #{Pipeline::Base.find(id).status}")
|
data/examples/helper.rb
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
gem 'activerecord'
|
3
|
+
gem 'collectiveidea-delayed_job', :lib => 'delayed_job'
|
4
|
+
|
5
|
+
require File.join(File.dirname(__FILE__), '..', 'init')
|
6
|
+
require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
|
7
|
+
ActiveRecord::Base.logger = Logger.new(STDOUT)
|
@@ -1,6 +1,4 @@
|
|
1
|
-
require File.join(File.dirname(__FILE__), '
|
2
|
-
require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
|
3
|
-
ActiveRecord::Base.logger = Logger.new(STDOUT)
|
1
|
+
require File.join(File.dirname(__FILE__), 'helper')
|
4
2
|
|
5
3
|
class Step1 < Pipeline::Stage::Base
|
6
4
|
def run
|
@@ -24,4 +22,4 @@ end
|
|
24
22
|
|
25
23
|
Pipeline.start(TwoStepPipeline.new)
|
26
24
|
|
27
|
-
Delayed::
|
25
|
+
Delayed::Job.work_off
|
@@ -1,6 +1,4 @@
|
|
1
|
-
require File.join(File.dirname(__FILE__), '
|
2
|
-
require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
|
3
|
-
ActiveRecord::Base.logger = Logger.new(STDOUT)
|
1
|
+
require File.join(File.dirname(__FILE__), 'helper')
|
4
2
|
|
5
3
|
class Step1 < Pipeline::Stage::Base
|
6
4
|
def run
|
@@ -28,6 +26,8 @@ end
|
|
28
26
|
|
29
27
|
id = Pipeline.start(TwoStepPipeline.new)
|
30
28
|
|
29
|
+
Delayed::Job.work_off
|
30
|
+
|
31
31
|
Pipeline.resume(id)
|
32
32
|
|
33
|
-
Delayed::
|
33
|
+
Delayed::Job.work_off
|
data/lib/pipeline/api_methods.rb
CHANGED
data/lib/pipeline/base.rb
CHANGED
@@ -5,7 +5,7 @@ module Pipeline
|
|
5
5
|
# :not_started ---> :in_progress ---> :completed
|
6
6
|
# ^ | \-> :failed
|
7
7
|
# | v
|
8
|
-
#
|
8
|
+
# :paused / :retry
|
9
9
|
symbol_attr :status
|
10
10
|
transactional_attr :status
|
11
11
|
private :status=
|
@@ -28,8 +28,8 @@ module Pipeline
|
|
28
28
|
end
|
29
29
|
|
30
30
|
def after_initialize
|
31
|
-
self[:status] = :not_started if new_record?
|
32
31
|
if new_record?
|
32
|
+
self[:status] = :not_started
|
33
33
|
self.class.defined_stages.each do |stage_class|
|
34
34
|
stages << stage_class.new(:pipeline => self)
|
35
35
|
end
|
@@ -51,9 +51,10 @@ module Pipeline
|
|
51
51
|
if e.input_required?
|
52
52
|
self.status = :paused
|
53
53
|
else
|
54
|
+
self.status = :retry
|
54
55
|
raise e
|
55
56
|
end
|
56
|
-
rescue
|
57
|
+
rescue Exception
|
57
58
|
self.status = (failure_mode == :cancel ? :failed : :paused)
|
58
59
|
end
|
59
60
|
end
|
@@ -64,7 +65,7 @@ module Pipeline
|
|
64
65
|
end
|
65
66
|
|
66
67
|
def ok_to_resume?
|
67
|
-
[:not_started, :paused].include?(status)
|
68
|
+
[:not_started, :paused, :retry].include?(status)
|
68
69
|
end
|
69
70
|
|
70
71
|
private
|
data/lib/pipeline/stage/base.rb
CHANGED
@@ -28,8 +28,10 @@ module Pipeline
|
|
28
28
|
class_inheritable_accessor :default_name, :instance_writer => false
|
29
29
|
|
30
30
|
def after_initialize
|
31
|
-
|
32
|
-
|
31
|
+
if new_record?
|
32
|
+
self[:status] = :not_started
|
33
|
+
self.name ||= (default_name || self.class).to_s
|
34
|
+
end
|
33
35
|
end
|
34
36
|
|
35
37
|
def completed?
|
@@ -43,7 +45,7 @@ module Pipeline
|
|
43
45
|
_setup
|
44
46
|
run
|
45
47
|
self.status = :completed
|
46
|
-
rescue => e
|
48
|
+
rescue Exception => e
|
47
49
|
logger.info("Error on stage #{default_name}: #{e.message}")
|
48
50
|
logger.info(e.backtrace.join("\n"))
|
49
51
|
self.message = e.message
|
@@ -58,6 +60,7 @@ module Pipeline
|
|
58
60
|
private
|
59
61
|
def _setup
|
60
62
|
self.attempts += 1
|
63
|
+
self.message = nil
|
61
64
|
self.status = :in_progress
|
62
65
|
end
|
63
66
|
end
|
data/pipeline.gemspec
CHANGED
@@ -2,11 +2,11 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = %q{pipeline}
|
5
|
-
s.version = "0.0.
|
5
|
+
s.version = "0.0.8"
|
6
6
|
|
7
7
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
8
|
s.authors = ["Danilo Sato"]
|
9
|
-
s.date = %q{2009-08-
|
9
|
+
s.date = %q{2009-08-20}
|
10
10
|
s.description = %q{Pipeline is a Rails plugin/gem to run asynchronous processes in a configurable pipeline.}
|
11
11
|
s.email = %q{danilo@dtsato.com}
|
12
12
|
s.extra_rdoc_files = [
|
@@ -22,6 +22,7 @@ Gem::Specification.new do |s|
|
|
22
22
|
"VERSION",
|
23
23
|
"examples/auto_recoverable_pipeline.rb",
|
24
24
|
"examples/cancelling_pipeline.rb",
|
25
|
+
"examples/helper.rb",
|
25
26
|
"examples/two_step_pipeline.rb",
|
26
27
|
"examples/user_recoverable_pipeline.rb",
|
27
28
|
"generators/pipeline/pipeline_generator.rb",
|
@@ -8,7 +8,7 @@ module Pipeline
|
|
8
8
|
describe "#start" do
|
9
9
|
before(:each) do
|
10
10
|
@pipeline = FakePipeline.new
|
11
|
-
@pipeline.stub!(:
|
11
|
+
@pipeline.stub!(:new_record?).and_return(false)
|
12
12
|
Delayed::Job.stub!(:enqueue)
|
13
13
|
end
|
14
14
|
|
@@ -17,12 +17,20 @@ module Pipeline
|
|
17
17
|
lambda {Pipeline.start(Object.new)}.should raise_error(InvalidPipelineError, "Invalid pipeline")
|
18
18
|
end
|
19
19
|
|
20
|
-
it "should save pipeline instance" do
|
20
|
+
it "should save pipeline instance (for new record)" do
|
21
|
+
@pipeline.should_receive(:new_record?).and_return(true)
|
21
22
|
@pipeline.should_receive(:save!)
|
22
23
|
|
23
24
|
Pipeline.start(@pipeline)
|
24
25
|
end
|
25
26
|
|
27
|
+
it "should not save pipeline instance (if already saved)" do
|
28
|
+
@pipeline.should_receive(:new_record?).and_return(false)
|
29
|
+
@pipeline.should_not_receive(:save!)
|
30
|
+
|
31
|
+
Pipeline.start(@pipeline)
|
32
|
+
end
|
33
|
+
|
26
34
|
it "should start a job for a pipeline instance" do
|
27
35
|
Delayed::Job.should_receive(:enqueue).with(@pipeline)
|
28
36
|
|
data/spec/pipeline/base_spec.rb
CHANGED
@@ -36,7 +36,7 @@ end
|
|
36
36
|
class GenericErrorStage < FirstStage
|
37
37
|
def run
|
38
38
|
super
|
39
|
-
raise
|
39
|
+
raise Exception.new
|
40
40
|
end
|
41
41
|
end
|
42
42
|
|
@@ -229,15 +229,15 @@ module Pipeline
|
|
229
229
|
lambda {@pipeline.perform}.should raise_error(RecoverableError)
|
230
230
|
end
|
231
231
|
|
232
|
-
it "should
|
232
|
+
it "should change status to :retry" do
|
233
233
|
lambda {@pipeline.perform}.should raise_error(RecoverableError)
|
234
|
-
@pipeline.status.should == :
|
234
|
+
@pipeline.status.should == :retry
|
235
235
|
end
|
236
236
|
|
237
237
|
it "should save status" do
|
238
238
|
@pipeline.save!
|
239
239
|
lambda {@pipeline.perform}.should raise_error(RecoverableError)
|
240
|
-
@pipeline.reload.status.should == :
|
240
|
+
@pipeline.reload.status.should == :retry
|
241
241
|
end
|
242
242
|
end
|
243
243
|
|
@@ -274,7 +274,7 @@ module Pipeline
|
|
274
274
|
end
|
275
275
|
|
276
276
|
it "should not re-raise error" do
|
277
|
-
lambda {@pipeline.perform}.should_not raise_error(
|
277
|
+
lambda {@pipeline.perform}.should_not raise_error(Exception)
|
278
278
|
end
|
279
279
|
|
280
280
|
it "should update status (pause mode)" do
|
@@ -368,6 +368,13 @@ module Pipeline
|
|
368
368
|
@pipeline.should be_ok_to_resume
|
369
369
|
lambda {@pipeline.perform}.should_not raise_error(InvalidStatusError)
|
370
370
|
end
|
371
|
+
|
372
|
+
it "should execute if status is :retry" do
|
373
|
+
@pipeline.send(:status=, :retry)
|
374
|
+
|
375
|
+
@pipeline.should be_ok_to_resume
|
376
|
+
lambda {@pipeline.perform}.should_not raise_error(InvalidStatusError)
|
377
|
+
end
|
371
378
|
|
372
379
|
it "should not execute if status is :in_progress" do
|
373
380
|
@pipeline.send(:status=, :in_progress)
|
@@ -173,6 +173,13 @@ module Pipeline
|
|
173
173
|
@stage.reload.message.should == "message"
|
174
174
|
end
|
175
175
|
|
176
|
+
it "should capture generic Exception" do
|
177
|
+
@stage.should_receive(:run).and_raise(Exception.new)
|
178
|
+
lambda {@stage.perform}.should raise_error(Exception)
|
179
|
+
@stage.status.should == :failed
|
180
|
+
@stage.reload.status.should == :failed
|
181
|
+
end
|
182
|
+
|
176
183
|
it "should log exception message and backtrace" do
|
177
184
|
SampleStage.default_name = "SampleStage"
|
178
185
|
error = StandardError.new("error message")
|
@@ -209,6 +216,14 @@ module Pipeline
|
|
209
216
|
stage.status.should == :in_progress
|
210
217
|
stage.reload.status.should == :in_progress
|
211
218
|
end
|
219
|
+
|
220
|
+
it "should clear message when restarting" do
|
221
|
+
stage = SampleStage.new(:message => 'some message')
|
222
|
+
stage.send(:_setup)
|
223
|
+
|
224
|
+
stage.message.should be_nil
|
225
|
+
stage.reload.message.should be_nil
|
226
|
+
end
|
212
227
|
end
|
213
228
|
|
214
229
|
describe "- execution (state transitions)" do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dtsato-pipeline
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Danilo Sato
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-08-
|
12
|
+
date: 2009-08-20 00:00:00 -07:00
|
13
13
|
default_executable:
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
@@ -50,6 +50,7 @@ files:
|
|
50
50
|
- VERSION
|
51
51
|
- examples/auto_recoverable_pipeline.rb
|
52
52
|
- examples/cancelling_pipeline.rb
|
53
|
+
- examples/helper.rb
|
53
54
|
- examples/two_step_pipeline.rb
|
54
55
|
- examples/user_recoverable_pipeline.rb
|
55
56
|
- generators/pipeline/pipeline_generator.rb
|