dtsato-pipeline 0.0.7 → 0.0.8

data/CHANGELOG

@@ -1,3 +1,12 @@
+0.0.8
+=====
+
+Bug Fix:
+* Clearing message when restarting a failed stage
+* Only saves the pipeline if it's a new record (client code is free to pass an already saved pipeline if needed)
+* Capturing generic Exception instead of StandardError (in case of Interrupts, Timeouts, ...)
+* Auto-recovering from errors was failing on InvalidStatusError. Added a new :retry status to differentiate between a paused or auto-recoverable error
+
 0.0.7
 =====
 

data/VERSION

@@ -1 +1 @@
-0.0.7
+0.0.8

data/examples/auto_recoverable_pipeline.rb

@@ -1,6 +1,4 @@
-require File.join(File.dirname(__FILE__), '..', 'init')
-require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
-ActiveRecord::Base.logger = Logger.new(STDOUT)
+require File.join(File.dirname(__FILE__), 'helper')
 
 class Step1 < Pipeline::Stage::Base
   def run
@@ -32,4 +30,7 @@ end
 
 Pipeline.start(TwoStepPipeline.new)
 
-Delayed::Worker.new.start
+Delayed::Job.work_off
+# Waiting for second job to pass re-scheduling time limit
+sleep(10)
+Delayed::Job.work_off

data/examples/cancelling_pipeline.rb

@@ -1,6 +1,4 @@
-require File.join(File.dirname(__FILE__), '..', 'init')
-require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
-ActiveRecord::Base.logger = Logger.new(STDOUT)
+require File.join(File.dirname(__FILE__), 'helper')
 
 class Step1 < Pipeline::Stage::Base
   def run
@@ -24,9 +22,7 @@ end
 
 id = Pipeline.start(TwoStepPipeline.new)
 
-Delayed::Worker.new.start
+Delayed::Job.work_off
 
-# CTRL-C to execute the cancelling, since we want to cancel after the stage failed, but
-# Worker is blocking the process on the previous line
 Pipeline.cancel(id)
-p Pipeline::Base.find(id)
+puts("Pipeline is now #{Pipeline::Base.find(id).status}")

data/examples/helper.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+gem 'activerecord'
+gem 'collectiveidea-delayed_job', :lib => 'delayed_job'
+
+require File.join(File.dirname(__FILE__), '..', 'init')
+require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
+ActiveRecord::Base.logger = Logger.new(STDOUT)

data/examples/two_step_pipeline.rb

@@ -1,6 +1,4 @@
-require File.join(File.dirname(__FILE__), '..', 'init')
-require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
-ActiveRecord::Base.logger = Logger.new(STDOUT)
+require File.join(File.dirname(__FILE__), 'helper')
 
 class Step1 < Pipeline::Stage::Base
   def run
@@ -24,4 +22,4 @@ end
 
 Pipeline.start(TwoStepPipeline.new)
 
-Delayed::Worker.new.start
+Delayed::Job.work_off

data/examples/user_recoverable_pipeline.rb

@@ -1,6 +1,4 @@
-require File.join(File.dirname(__FILE__), '..', 'init')
-require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
-ActiveRecord::Base.logger = Logger.new(STDOUT)
+require File.join(File.dirname(__FILE__), 'helper')
 
 class Step1 < Pipeline::Stage::Base
   def run
@@ -28,6 +26,8 @@ end
 
 id = Pipeline.start(TwoStepPipeline.new)
 
+Delayed::Job.work_off
+
 Pipeline.resume(id)
 
-Delayed::Worker.new.start
+Delayed::Job.work_off

data/lib/pipeline/api_methods.rb

@@ -2,7 +2,7 @@ module Pipeline
   module ApiMethods
     def start(pipeline)
       raise InvalidPipelineError.new("Invalid pipeline") unless pipeline.is_a?(Pipeline::Base)
-      pipeline.save!
+      pipeline.save! if pipeline.new_record?
       Delayed::Job.enqueue(pipeline)
       pipeline.id
     end

data/lib/pipeline/base.rb

@@ -5,7 +5,7 @@ module Pipeline
     # :not_started ---> :in_progress ---> :completed
     #                       ^ |       \-> :failed
     #                       | v
-    #                     :paused
+    #                 :paused / :retry
     symbol_attr :status
     transactional_attr :status
     private :status=
@@ -28,8 +28,8 @@ module Pipeline
     end
 
     def after_initialize
-      self[:status] = :not_started if new_record?
       if new_record?
+        self[:status] = :not_started
         self.class.defined_stages.each do |stage_class|
           stages << stage_class.new(:pipeline => self)
         end
@@ -51,9 +51,10 @@ module Pipeline
         if e.input_required?
           self.status = :paused
         else
+          self.status = :retry
           raise e
         end
-      rescue
+      rescue Exception
         self.status = (failure_mode == :cancel ? :failed : :paused)
       end
     end
@@ -64,7 +65,7 @@ module Pipeline
     end
     
     def ok_to_resume?
-      [:not_started, :paused].include?(status)
+      [:not_started, :paused, :retry].include?(status)
     end
 
     private

data/lib/pipeline/stage/base.rb

@@ -28,8 +28,10 @@ module Pipeline
       class_inheritable_accessor :default_name, :instance_writer => false
       
       def after_initialize
-        self.name ||= (default_name || self.class).to_s
-        self[:status] = :not_started if new_record?
+        if new_record?
+          self[:status] = :not_started
+          self.name ||= (default_name || self.class).to_s
+        end
       end
       
       def completed?
@@ -43,7 +45,7 @@ module Pipeline
           _setup
           run
           self.status = :completed
-        rescue => e
+        rescue Exception => e
           logger.info("Error on stage #{default_name}: #{e.message}")
           logger.info(e.backtrace.join("\n"))
           self.message = e.message
@@ -58,6 +60,7 @@ module Pipeline
       private
       def _setup
         self.attempts += 1
+        self.message = nil
         self.status = :in_progress
       end
     end

data/pipeline.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{pipeline}
-  s.version = "0.0.7"
+  s.version = "0.0.8"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Danilo Sato"]
-  s.date = %q{2009-08-18}
+  s.date = %q{2009-08-20}
   s.description = %q{Pipeline is a Rails plugin/gem to run asynchronous processes in a configurable pipeline.}
   s.email = %q{danilo@dtsato.com}
   s.extra_rdoc_files = [
@@ -22,6 +22,7 @@ Gem::Specification.new do |s|
      "VERSION",
      "examples/auto_recoverable_pipeline.rb",
      "examples/cancelling_pipeline.rb",
+     "examples/helper.rb",
      "examples/two_step_pipeline.rb",
      "examples/user_recoverable_pipeline.rb",
      "generators/pipeline/pipeline_generator.rb",

data/spec/pipeline/api_methods_spec.rb

@@ -8,7 +8,7 @@ module Pipeline
     describe "#start" do
       before(:each) do
         @pipeline = FakePipeline.new
-        @pipeline.stub!(:save!)
+        @pipeline.stub!(:new_record?).and_return(false)
         Delayed::Job.stub!(:enqueue)
       end
       
@@ -17,12 +17,20 @@ module Pipeline
         lambda {Pipeline.start(Object.new)}.should raise_error(InvalidPipelineError, "Invalid pipeline")
       end
 
-      it "should save pipeline instance" do
+      it "should save pipeline instance (for new record)" do
+        @pipeline.should_receive(:new_record?).and_return(true)
         @pipeline.should_receive(:save!)
         
         Pipeline.start(@pipeline)
       end
 
+      it "should not save pipeline instance (if already saved)" do
+        @pipeline.should_receive(:new_record?).and_return(false)
+        @pipeline.should_not_receive(:save!)
+        
+        Pipeline.start(@pipeline)
+      end
+
       it "should start a job for a pipeline instance" do
         Delayed::Job.should_receive(:enqueue).with(@pipeline)
         

data/spec/pipeline/base_spec.rb

@@ -36,7 +36,7 @@ end
 class GenericErrorStage < FirstStage
   def run
     super
-    raise StandardError.new
+    raise Exception.new
   end
 end
 
@@ -229,15 +229,15 @@ module Pipeline
         lambda {@pipeline.perform}.should raise_error(RecoverableError)
       end
       
-      it "should keep status :in_progress" do
+      it "should change status to :retry" do
         lambda {@pipeline.perform}.should raise_error(RecoverableError)
-        @pipeline.status.should == :in_progress
+        @pipeline.status.should == :retry
       end
       
       it "should save status" do
         @pipeline.save!
         lambda {@pipeline.perform}.should raise_error(RecoverableError)
-        @pipeline.reload.status.should == :in_progress
+        @pipeline.reload.status.should == :retry
       end
     end
 
@@ -274,7 +274,7 @@ module Pipeline
       end
 
       it "should not re-raise error" do
-        lambda {@pipeline.perform}.should_not raise_error(StandardError)
+        lambda {@pipeline.perform}.should_not raise_error(Exception)
       end
       
       it "should update status (pause mode)" do
@@ -368,6 +368,13 @@ module Pipeline
         @pipeline.should be_ok_to_resume
         lambda {@pipeline.perform}.should_not raise_error(InvalidStatusError)
       end
+
+      it "should execute if status is :retry" do
+        @pipeline.send(:status=, :retry)
+        
+        @pipeline.should be_ok_to_resume
+        lambda {@pipeline.perform}.should_not raise_error(InvalidStatusError)
+      end
       
       it "should not execute if status is :in_progress" do
         @pipeline.send(:status=, :in_progress)

data/spec/pipeline/stage/base_spec.rb

@@ -173,6 +173,13 @@ module Pipeline
           @stage.reload.message.should == "message"
         end
         
+        it "should capture generic Exception" do
+          @stage.should_receive(:run).and_raise(Exception.new)
+          lambda {@stage.perform}.should raise_error(Exception)
+          @stage.status.should == :failed
+          @stage.reload.status.should == :failed
+        end
+        
         it "should log exception message and backtrace" do
           SampleStage.default_name = "SampleStage"
           error = StandardError.new("error message")
@@ -209,6 +216,14 @@ module Pipeline
           stage.status.should == :in_progress
           stage.reload.status.should == :in_progress
         end
+
+        it "should clear message when restarting" do
+          stage = SampleStage.new(:message => 'some message')
+          stage.send(:_setup)
+          
+          stage.message.should be_nil
+          stage.reload.message.should be_nil
+        end
       end
       
       describe "- execution (state transitions)" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: dtsato-pipeline
 version: !ruby/object:Gem::Version 
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors: 
 - Danilo Sato
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-08-18 00:00:00 -07:00
+date: 2009-08-20 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -50,6 +50,7 @@ files:
 - VERSION
 - examples/auto_recoverable_pipeline.rb
 - examples/cancelling_pipeline.rb
+- examples/helper.rb
 - examples/two_step_pipeline.rb
 - examples/user_recoverable_pipeline.rb
 - generators/pipeline/pipeline_generator.rb