dtsato-pipeline 0.0.6 → 0.0.7

data/CHANGELOG

@@ -1,3 +1,16 @@
+0.0.7
+=====
+
+Features:
+* Allowing for user-defined default failure mode: when generic errors are raised, pipeline can be paused (default) or fail. Usage:
+  class MySamplePipeline < Pipeline::Base
+    self.default_failure_mode = :cancel
+  end
+
+Bug Fix:
+* Refreshes the pipeline and stage statuses before execution (at runtime) in case of offline status update
+* Rescue from ActiveRecord::RecordNotFound instead of relying on ActiveRecord#find(id) returning nil
+
 0.0.6
 =====
 

data/Rakefile

@@ -77,7 +77,7 @@ begin
   Synthesis::Task.new("spec:synthesis") do |t|
     t.adapter = :rspec
     t.pattern = 'spec/**/*_spec.rb'
-    t.ignored = ['Pipeline::FakePipeline', 'Delayed::Job']
+    t.ignored = ['Pipeline::FakePipeline', 'Delayed::Job', 'SampleStage', 'Logger']
   end
 rescue LoadError
   desc 'Synthesis rake task not available'

data/TODO

@@ -4,6 +4,7 @@
 * Improve documentation (RDoc)
 * Adapter for other persistence frameworks (ActiveModel?)
 * Adapter for other background processing mechanisms (currently relying on dj's auto-retry)
+* Max attempts on pipeline (or stage)
 
 Think more about:
 - Repeated code on status handling on pipeline instance/pipeline stage

data/VERSION

@@ -1 +1 @@
-0.0.6
+0.0.7

data/lib/pipeline/api_methods.rb

@@ -9,15 +9,17 @@ module Pipeline
     
     def resume(id)
       pipeline = Base.find(id)
-      raise InvalidPipelineError.new("Invalid pipeline") unless pipeline
       raise InvalidStatusError.new(pipeline.status) unless pipeline.ok_to_resume?
       Delayed::Job.enqueue(pipeline)
+    rescue ActiveRecord::RecordNotFound
+      raise InvalidPipelineError.new("Invalid pipeline")
     end
     
     def cancel(id)
       pipeline = Base.find(id)
-      raise InvalidPipelineError.new("Invalid pipeline") unless pipeline
       pipeline.cancel
+    rescue ActiveRecord::RecordNotFound
+      raise InvalidPipelineError.new("Invalid pipeline")
     end
   end
 end

data/lib/pipeline/base.rb

@@ -14,11 +14,19 @@ module Pipeline
 
     class_inheritable_accessor :defined_stages, :instance_writer => false
     self.defined_stages = []
+
+    class_inheritable_accessor :failure_mode, :instance_writer => false
+    self.failure_mode = :pause
     
     def self.define_stages(stages)
       self.defined_stages = stages.build_chain
     end
 
+    def self.default_failure_mode=(mode)
+      new_mode = [:pause, :cancel].include?(mode) ? mode : :pause
+      self.failure_mode = new_mode
+    end
+
     def after_initialize
       self[:status] = :not_started if new_record?
       if new_record?
@@ -29,6 +37,7 @@ module Pipeline
     end
     
     def perform
+      reload unless new_record?
       raise InvalidStatusError.new(status) unless ok_to_resume?
       begin
         _setup
@@ -45,7 +54,7 @@ module Pipeline
           raise e
         end
       rescue
-        self.status = :paused
+        self.status = (failure_mode == :cancel ? :failed : :paused)
       end
     end
     

data/lib/pipeline/stage/base.rb

@@ -37,6 +37,7 @@ module Pipeline
       end
       
       def perform
+        reload unless new_record?
         raise InvalidStatusError.new(status) unless [:not_started, :failed].include?(status)
         begin
           _setup

data/pipeline.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{pipeline}
-  s.version = "0.0.6"
+  s.version = "0.0.7"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Danilo Sato"]
-  s.date = %q{2009-08-12}
+  s.date = %q{2009-08-18}
   s.description = %q{Pipeline is a Rails plugin/gem to run asynchronous processes in a configurable pipeline.}
   s.email = %q{danilo@dtsato.com}
   s.extra_rdoc_files = [

data/spec/pipeline/api_methods_spec.rb

@@ -50,8 +50,10 @@ module Pipeline
         Pipeline.resume('1')
       end
 
-      it "should raise error is trying to resume invalid pipeline" do
-        Pipeline::Base.should_receive(:find).and_return(nil)
+      it "should raise error if trying to resume invalid pipeline" do
+        Pipeline::Base.should_receive(:find).
+          with('1').
+          and_raise(ActiveRecord::RecordNotFound.new)
 
         lambda {Pipeline.resume('1')}.should raise_error(InvalidPipelineError, "Invalid pipeline")
       end
@@ -83,7 +85,9 @@ module Pipeline
       end
 
       it "should raise error is trying to cancel invalid pipeline" do
-        Pipeline::Base.should_receive(:find).and_return(nil)
+        Pipeline::Base.should_receive(:find).
+          with('1').
+          and_raise(ActiveRecord::RecordNotFound.new)
 
         lambda {Pipeline.cancel('1')}.should raise_error(InvalidPipelineError, "Invalid pipeline")
       end

data/spec/pipeline/base_spec.rb

@@ -12,6 +12,34 @@ end
 
 class SecondStage < FirstStage; end # Ugly.. just so I don't have to write stub again
 
+class IrrecoverableStage < FirstStage
+  def run
+    super
+    raise Pipeline::IrrecoverableError.new
+  end
+end
+
+class RecoverableInputRequiredStage < FirstStage
+  def run
+    super
+    raise Pipeline::RecoverableError.new("message", true)
+  end
+end
+
+class RecoverableStage < FirstStage
+  def run
+    super
+    raise Pipeline::RecoverableError.new("message")
+  end
+end
+
+class GenericErrorStage < FirstStage
+  def run
+    super
+    raise StandardError.new
+  end
+end
+
 class SamplePipeline < Pipeline::Base
   define_stages FirstStage >> SecondStage
 end
@@ -20,9 +48,26 @@ module Pipeline
   describe Base do
 
     describe "- configuring" do
+      before(:each) do
+        class ::SamplePipeline
+          define_stages FirstStage >> SecondStage
+        end
+      end
+      
       it "should allow accessing stages" do
         SamplePipeline.defined_stages.should == [FirstStage, SecondStage]
       end
+      
+      it "should allow configuring default failure mode (pause by default)" do
+        SamplePipeline.default_failure_mode = :pause
+        SamplePipeline.failure_mode.should == :pause
+        
+        SamplePipeline.default_failure_mode = :cancel
+        SamplePipeline.failure_mode.should == :cancel
+        
+        SamplePipeline.default_failure_mode = :something_else
+        SamplePipeline.failure_mode.should == :pause
+      end
     end
     
     describe "- setup" do
@@ -59,6 +104,8 @@ module Pipeline
         
         retrieved_pipeline = Base.find(@pipeline.id.to_s)
         retrieved_pipeline.should === @pipeline
+        
+        lambda {Base.find('invalid_id')}.should raise_error(ActiveRecord::RecordNotFound)
       end
 
       it "should persist type as single table inheritance" do
@@ -104,21 +151,18 @@ module Pipeline
     
     describe "- execution (success)" do
       before(:each) do
-        @pipeline = SamplePipeline.new
+        class ::SamplePipeline
+          define_stages FirstStage >> SecondStage
+        end
+        @pipeline = ::SamplePipeline.new
       end
 
       it "should increment attempts" do
-        failed_stage = SecondStage.new
-        failed_stage.stub!(:run).and_raise(RecoverableError.new("message", true))
-        SecondStage.stub!(:new).and_return(failed_stage)
-        
         pipeline = SamplePipeline.new
         
+        pipeline.attempts.should == 0
         pipeline.perform
         pipeline.attempts.should == 1
-
-        pipeline.perform
-        pipeline.attempts.should == 2
       end
       
       it "should perform each stage" do
@@ -151,10 +195,10 @@ module Pipeline
     
     describe "- execution (irrecoverable error)" do
       before(:each) do
-        failed_stage = SecondStage.new
-        failed_stage.stub!(:run).and_raise(IrrecoverableError.new)
-        SecondStage.stub!(:new).and_return(failed_stage)
-        @pipeline = SamplePipeline.new
+        class ::SamplePipeline
+          define_stages FirstStage >> IrrecoverableStage
+        end
+        @pipeline = ::SamplePipeline.new
       end
 
       it "should not re-raise error" do
@@ -175,9 +219,9 @@ module Pipeline
     
     describe "- execution (recoverable error that doesn't require user input)" do
       before(:each) do
-        failed_stage = SecondStage.new
-        failed_stage.stub!(:run).and_raise(RecoverableError.new)
-        SecondStage.stub!(:new).and_return(failed_stage)
+        class ::SamplePipeline
+          define_stages FirstStage >> RecoverableStage
+        end
         @pipeline = SamplePipeline.new
       end
 
@@ -199,9 +243,9 @@ module Pipeline
 
     describe "- execution (recoverable error that requires user input)" do
       before(:each) do
-        failed_stage = SecondStage.new
-        failed_stage.stub!(:run).and_raise(RecoverableError.new('message', true))
-        SecondStage.stub!(:new).and_return(failed_stage)
+        class ::SamplePipeline
+          define_stages FirstStage >> RecoverableInputRequiredStage
+        end
         @pipeline = SamplePipeline.new
       end
 
@@ -221,11 +265,11 @@ module Pipeline
       end
     end
 
-    describe "- execution (other errors will pause the pipeline)" do
+    describe "- execution (other errors will use failure mode to pause/cancel pipeline)" do
       before(:each) do
-        failed_stage = SecondStage.new
-        failed_stage.stub!(:run).and_raise(StandardError.new)
-        SecondStage.stub!(:new).and_return(failed_stage)
+        class ::SamplePipeline
+          define_stages FirstStage >> GenericErrorStage
+        end
         @pipeline = SamplePipeline.new
       end
 
@@ -233,26 +277,38 @@ module Pipeline
         lambda {@pipeline.perform}.should_not raise_error(StandardError)
       end
       
-      it "should update status" do
+      it "should update status (pause mode)" do
+        SamplePipeline.default_failure_mode = :pause
         @pipeline.perform
         @pipeline.status.should == :paused
       end
       
-      it "should save status" do
+      it "should save status (pause mode)" do
+        SamplePipeline.default_failure_mode = :pause
         @pipeline.save!
         @pipeline.perform
         @pipeline.reload.status.should == :paused
       end
+
+      it "should update status (cancel mode)" do
+        SamplePipeline.default_failure_mode = :cancel
+        @pipeline.perform
+        @pipeline.status.should == :failed
+      end
+      
+      it "should save status (cancel mode)" do
+        SamplePipeline.default_failure_mode = :cancel
+        @pipeline.save!
+        @pipeline.perform
+        @pipeline.reload.status.should == :failed
+      end
     end
 
     describe "- execution (retrying)" do
       before(:each) do
-        @passing_stage = FirstStage.new
-        FirstStage.stub!(:new).and_return(@passing_stage)
-        
-        @failed_stage = SecondStage.new
-        @failed_stage.stub!(:run).and_raise(RecoverableError.new('message', true))
-        SecondStage.stub!(:new).and_return(@failed_stage)
+        class ::SamplePipeline
+          define_stages FirstStage >> RecoverableInputRequiredStage
+        end
         @pipeline = SamplePipeline.new
       end
 
@@ -273,12 +329,26 @@ module Pipeline
       
       it "should skip completed stages" do
         @pipeline.perform
-        @passing_stage.attempts.should == 1
-        @failed_stage.attempts.should == 1
+        @pipeline.stages[0].attempts.should == 1
+        @pipeline.stages[1].attempts.should == 1
         
         @pipeline.perform
-        @passing_stage.attempts.should == 1
-        @failed_stage.attempts.should == 2
+        @pipeline.stages[0].attempts.should == 1
+        @pipeline.stages[1].attempts.should == 2
+      end
+      
+      it "should refresh object (in case it was cancelled after job was scheduled)" do
+        # Gets paused on the first time
+        @pipeline.save!
+        @pipeline.perform
+        
+        # Status gets updated to failed on the database (not on the current instance)
+        same_pipeline = SamplePipeline.find(@pipeline.id)
+        same_pipeline.send(:status=, :failed)
+        same_pipeline.save!
+        
+        # Retrying should fail because pipeline is now failed
+        lambda {@pipeline.perform}.should raise_error(InvalidStatusError, "Status is already failed")
       end
     end
     
@@ -323,12 +393,9 @@ module Pipeline
     
     describe "- cancelling" do
       before(:each) do
-        @passing_stage = FirstStage.new
-        FirstStage.stub!(:new).and_return(@passing_stage)
-        
-        @failed_stage = SecondStage.new
-        @failed_stage.stub!(:run).and_raise(RecoverableError.new('message', true))
-        SecondStage.stub!(:new).and_return(@failed_stage)
+        class ::SamplePipeline
+          define_stages FirstStage >> RecoverableInputRequiredStage
+        end
         @pipeline = SamplePipeline.new
         @pipeline.perform
       end

data/spec/pipeline/stage/base_spec.rb

@@ -76,6 +76,8 @@ module Pipeline
 
           s = SampleStage.find(@stage.id)
           s.should === @stage
+          
+          lambda {Base.find('invalid_id')}.should raise_error(ActiveRecord::RecordNotFound)          
         end
 
         it "should persist type as single table inheritance" do
@@ -113,12 +115,9 @@ module Pipeline
         end
         
         it "should increment attempts" do
-          @stage.stub!(:run).and_raise(StandardError.new)
-          lambda {@stage.perform}.should raise_error(StandardError)
+          @stage.attempts.should == 0
+          @stage.perform
           @stage.attempts.should == 1
-
-          lambda {@stage.perform}.should raise_error(StandardError)
-          @stage.attempts.should == 2
         end
         
         it "should call template method #run" do
@@ -126,6 +125,7 @@ module Pipeline
           @stage.perform
           @stage.should be_executed
         end
+                
       end
       
       describe "- execution (failure)" do
@@ -184,6 +184,21 @@ module Pipeline
           lambda {@stage.perform}.should raise_error
         end
 
+        it "should refresh object (in case it was cancelled after job was scheduled)" do
+          # Gets failed on the first time
+          @stage.save!
+          @stage.should_receive(:run).and_raise(RecoverableError.new("message"))
+          lambda {@stage.perform}.should raise_error(RecoverableError)
+
+          # Status gets updated to completed on the database (not on the current instance)
+          same_stage = SampleStage.find(@stage.id)
+          same_stage.send(:status=, :completed)
+          same_stage.save!
+
+          # Retrying should fail because stage is now completed
+          lambda {@stage.perform}.should raise_error(InvalidStatusError, "Status is already completed")
+        end
+
       end
       
       describe "- execution (in progress)" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: dtsato-pipeline
 version: !ruby/object:Gem::Version 
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors: 
 - Danilo Sato
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-08-12 00:00:00 -07:00
+date: 2009-08-18 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency