dtsato-pipeline 0.0.1

data/pipeline.gemspec

@@ -0,0 +1,86 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{pipeline}
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Danilo Sato"]
+  s.date = %q{2009-07-30}
+  s.description = %q{Pipeline is a Rails plugin/gem to run asynchronous processes in a configurable pipeline.}
+  s.email = %q{danilo@dtsato.com}
+  s.extra_rdoc_files = [
+    "README.rdoc"
+  ]
+  s.files = [
+    ".gitignore",
+     "CHANGELOG",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "examples/auto_recoverable_pipeline.rb",
+     "examples/cancelling_pipeline.rb",
+     "examples/two_step_pipeline.rb",
+     "examples/user_recoverable_pipeline.rb",
+     "generators/pipeline/pipeline_generator.rb",
+     "generators/pipeline/templates/pipeline_instances_migration.rb",
+     "generators/pipeline/templates/pipeline_stages_migration.rb",
+     "init.rb",
+     "lib/pipeline.rb",
+     "lib/pipeline/api_methods.rb",
+     "lib/pipeline/base.rb",
+     "lib/pipeline/core_ext/symbol_attribute.rb",
+     "lib/pipeline/core_ext/transactional_attribute.rb",
+     "lib/pipeline/errors.rb",
+     "lib/pipeline/stage/base.rb",
+     "pipeline.gemspec",
+     "spec/database_integration_helper.rb",
+     "spec/pipeline/api_methods_spec.rb",
+     "spec/pipeline/base_spec.rb",
+     "spec/pipeline/core_ext/symbol_attribute_spec.rb",
+     "spec/pipeline/core_ext/transactional_attribute_spec.rb",
+     "spec/pipeline/errors_spec.rb",
+     "spec/pipeline/stage/base_spec.rb",
+     "spec/rcov.opts",
+     "spec/spec.opts",
+     "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/dtsato/pipeline}
+  s.rdoc_options = ["--main", "README.rdoc", "--inline-source", "--line-numbers"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{pipeline}
+  s.rubygems_version = %q{1.3.4}
+  s.summary = %q{A Rails plugin/gem to run asynchronous processes in a configurable pipeline}
+  s.test_files = [
+    "spec/database_integration_helper.rb",
+     "spec/pipeline",
+     "spec/pipeline/api_methods_spec.rb",
+     "spec/pipeline/base_spec.rb",
+     "spec/pipeline/core_ext",
+     "spec/pipeline/core_ext/symbol_attribute_spec.rb",
+     "spec/pipeline/core_ext/transactional_attribute_spec.rb",
+     "spec/pipeline/errors_spec.rb",
+     "spec/pipeline/stage",
+     "spec/pipeline/stage/base_spec.rb",
+     "spec/rcov.opts",
+     "spec/spec.opts",
+     "spec/spec_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<activerecord>, [">= 2.0"])
+      s.add_runtime_dependency(%q<collectiveidea-delayed_job>, [">= 1.8.0"])
+    else
+      s.add_dependency(%q<activerecord>, [">= 2.0"])
+      s.add_dependency(%q<collectiveidea-delayed_job>, [">= 1.8.0"])
+    end
+  else
+    s.add_dependency(%q<activerecord>, [">= 2.0"])
+    s.add_dependency(%q<collectiveidea-delayed_job>, [">= 1.8.0"])
+  end
+end

data/spec/database_integration_helper.rb

@@ -0,0 +1,42 @@
+require 'rubygems'
+gem 'sqlite3-ruby'
+
+ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => 'pipeline.sqlite')
+ActiveRecord::Migration.verbose = false
+
+ActiveRecord::Schema.define do
+
+  create_table :delayed_jobs, :force => true do |table|
+    table.integer  :priority, :default => 0
+    table.integer  :attempts, :default => 0
+    table.text     :handler
+    table.string   :last_error
+    table.datetime :run_at
+    table.datetime :locked_at
+    table.string   :locked_by
+    table.datetime :failed_at
+    table.timestamps
+  end
+
+  create_table :pipeline_instances, :force => true do |t|
+    t.string :type
+    t.string :status
+    t.integer :attempts, :default => 0
+    t.timestamps
+  end
+
+  create_table :pipeline_stages, :force => true do |t|
+    t.references :pipeline_instance
+    t.string  :type
+    t.string  :name
+    t.string  :status
+    t.text    :message
+    t.integer :attempts, :default => 0
+    t.timestamps
+  end
+
+end
+
+at_exit do
+  File.delete("pipeline.sqlite") if File.exists?("pipeline.sqlite")
+end

data/spec/pipeline/api_methods_spec.rb

@@ -0,0 +1,98 @@
+require File.join(File.dirname(__FILE__), '..', 'spec_helper')
+
+module Pipeline
+  describe ApiMethods do
+    class FakePipeline < Pipeline::Base
+    end
+  
+    describe "#start" do
+      before(:each) do
+        @pipeline = FakePipeline.new
+        @pipeline.stub!(:save!)
+        Delayed::Job.stub!(:enqueue)
+      end
+      
+      it "should only accept instance of Pipeline::Base" do
+        lambda {Pipeline.start(@pipeline)}.should_not raise_error
+        lambda {Pipeline.start(Object.new)}.should raise_error(InvalidPipelineError, "Invalid pipeline")
+      end
+
+      it "should save pipeline instance" do
+        @pipeline.should_receive(:save!)
+        
+        Pipeline.start(@pipeline)
+      end
+
+      it "should start a job for a pipeline instance" do
+        Delayed::Job.should_receive(:enqueue).with(@pipeline)
+        
+        Pipeline.start(@pipeline)
+      end
+    
+      it "should provide a token for the pipeline instance" do
+        @pipeline.stub!(:id).and_return('123')
+
+        token = Pipeline.start(@pipeline)
+        token.should == '123'
+      end
+    end
+
+    describe "#resume" do
+      before(:each) do
+        @pipeline = Pipeline::Base.new
+        Pipeline::Base.stub!(:find).with('1').and_return(@pipeline)
+        Delayed::Job.stub!(:enqueue)
+      end
+      
+      it "should accept a token for a pipeline instance" do
+        Pipeline::Base.should_receive(:find).with('1')
+
+        Pipeline.resume('1')
+      end
+
+      it "should raise error is trying to resume invalid pipeline" do
+        Pipeline::Base.should_receive(:find).and_return(nil)
+
+        lambda {Pipeline.resume('1')}.should raise_error(InvalidPipelineError, "Invalid pipeline")
+      end
+    
+      it "should start a new job for that pipeline instance" do
+        Delayed::Job.should_receive(:enqueue).with(@pipeline)
+        
+        Pipeline.resume('1')
+      end
+      
+      it "should raise error is trying to resume in invalid state" do
+        @pipeline.send(:_setup)
+
+        lambda {Pipeline.resume('1')}.should raise_error(InvalidStatusError, "Status is already in progress")
+      end
+
+    end
+
+    describe "#cancel" do
+      before(:each) do
+        @pipeline = Pipeline::Base.new
+        @pipeline.stub!(:cancel)
+        Pipeline::Base.stub!(:find).with('1').and_return(@pipeline)
+      end
+      
+      it "should accept a token for a pipeline instance" do
+        Pipeline::Base.should_receive(:find).with('1')
+        Pipeline.cancel('1')
+      end
+
+      it "should raise error is trying to cancel invalid pipeline" do
+        Pipeline::Base.should_receive(:find).and_return(nil)
+
+        lambda {Pipeline.cancel('1')}.should raise_error(InvalidPipelineError, "Invalid pipeline")
+      end
+    
+      it "should cancel pipeline instance" do
+        @pipeline.should_receive(:cancel)
+        Pipeline.cancel('1')
+      end
+    end
+
+  end
+end

data/spec/pipeline/base_spec.rb

@@ -0,0 +1,383 @@
+require File.join(File.dirname(__FILE__), '..', 'spec_helper')
+
+class FirstStage < Pipeline::Stage::Base
+  def run
+    @executed = true
+  end
+  
+  def executed?
+    !!@executed
+  end
+end
+
+class SecondStage < FirstStage; end # Ugly.. just so I don't have to write stub again
+
+class SamplePipeline < Pipeline::Base
+  define_stages FirstStage >> SecondStage
+end
+
+module Pipeline
+  describe Base do
+
+    describe "- configuring" do
+      it "should allow accessing stages" do
+        SamplePipeline.defined_stages.should == [FirstStage, SecondStage]
+      end
+    end
+    
+    describe "- setup" do
+      before(:each) do
+        @pipeline = SamplePipeline.new
+      end
+      
+      it "should start with status not_started" do
+        @pipeline.status.should == :not_started
+      end
+      
+      it "should instantiate stages with status not_started" do
+        @pipeline.stages.each { |stage| stage.status.should == :not_started }
+      end
+      
+      it "should validate status" do
+        lambda {Base.new(:status => :something_else)}.should raise_error
+      end
+    end
+
+    describe "- persistence" do
+      before(:each) do
+        @pipeline = Base.new
+      end
+      
+      it "should persist pipeline instance" do
+        @pipeline.id.should be_nil
+        lambda {@pipeline.save!}.should_not raise_error
+        @pipeline.id.should_not be_nil
+      end
+      
+      it "should allow retrieval by id" do
+        @pipeline.save!
+        
+        retrieved_pipeline = Base.find(@pipeline.id.to_s)
+        retrieved_pipeline.should === @pipeline
+      end
+
+      it "should persist type as single table inheritance" do
+        pipeline = SamplePipeline.new
+        pipeline.save!
+        
+        retrieved_pipeline = Base.find(pipeline.id)
+        retrieved_pipeline.should be_an_instance_of(SamplePipeline)
+      end
+      
+      it "should persist pipeline stages" do
+        pipeline = SamplePipeline.new
+        pipeline.stages.each {|stage| stage.id.should be_nil}
+        lambda {pipeline.save!}.should_not raise_error
+        pipeline.stages.each {|stage| stage.id.should_not be_nil}
+      end
+      
+      it "should allow retrieval of stages with pipeline instance" do
+        pipeline = SamplePipeline.new
+        pipeline.save!
+        
+        retrieved_pipeline = SamplePipeline.find(pipeline.id)
+        retrieved_pipeline.stages.should === pipeline.stages
+      end
+
+      it "should associate stages with pipeline instance" do
+        pipeline = SamplePipeline.new
+        pipeline.save!
+        
+        pipeline.stages.each {|stage| stage.pipeline.should === pipeline}
+      end
+      
+      it "should destroy stages when pipeline instance is destroyed" do
+        pipeline = SamplePipeline.new
+        pipeline.save!
+        
+        Pipeline::Stage::Base.count(:conditions => ['pipeline_instance_id = ?', pipeline.id]).should > 0
+        
+        pipeline.destroy
+        Pipeline::Stage::Base.count(:conditions => ['pipeline_instance_id = ?', pipeline.id]).should == 0
+      end
+    end
+    
+    describe "- execution (success)" do
+      before(:each) do
+        @pipeline = SamplePipeline.new
+      end
+
+      it "should increment attempts" do
+        failed_stage = SecondStage.new
+        failed_stage.stub!(:run).and_raise(RecoverableError.new("message", true))
+        SecondStage.stub!(:new).and_return(failed_stage)
+        
+        pipeline = SamplePipeline.new
+        
+        pipeline.perform
+        pipeline.attempts.should == 1
+
+        pipeline.perform
+        pipeline.attempts.should == 2
+      end
+      
+      it "should perform each stage" do
+        @pipeline.stages.each { |stage| stage.should_not be_executed }
+        @pipeline.perform
+        @pipeline.stages.each { |stage| stage.should be_executed }
+      end
+      
+      it "should update pipeline status after all stages finished" do
+        @pipeline.perform
+        @pipeline.status.should == :completed
+      end
+      
+      it "should save status" do
+        @pipeline.save!
+        @pipeline.perform
+        @pipeline.reload.status.should == :completed
+      end
+    end
+    
+    describe "- execution (in progress)" do
+      it "should set status to in_progress" do
+        pipeline = SamplePipeline.new
+        pipeline.send(:_setup)
+        
+        pipeline.status.should == :in_progress
+        pipeline.reload.status.should == :in_progress
+      end
+    end
+    
+    describe "- execution (irrecoverable error)" do
+      before(:each) do
+        failed_stage = SecondStage.new
+        failed_stage.stub!(:run).and_raise(IrrecoverableError.new)
+        SecondStage.stub!(:new).and_return(failed_stage)
+        @pipeline = SamplePipeline.new
+      end
+
+      it "should not re-raise error" do
+        lambda {@pipeline.perform}.should_not raise_error(IrrecoverableError)
+      end
+      
+      it "should update status" do
+        @pipeline.perform
+        @pipeline.status.should == :failed
+      end
+      
+      it "should save status" do
+        @pipeline.save!
+        @pipeline.perform
+        @pipeline.reload.status.should == :failed
+      end
+    end
+    
+    describe "- execution (recoverable error that doesn't require user input)" do
+      before(:each) do
+        failed_stage = SecondStage.new
+        failed_stage.stub!(:run).and_raise(RecoverableError.new)
+        SecondStage.stub!(:new).and_return(failed_stage)
+        @pipeline = SamplePipeline.new
+      end
+
+      it "should re-raise error (so delayed_job retry works)" do
+        lambda {@pipeline.perform}.should raise_error(RecoverableError)
+      end
+      
+      it "should keep status :in_progress" do
+        lambda {@pipeline.perform}.should raise_error(RecoverableError)
+        @pipeline.status.should == :in_progress
+      end
+      
+      it "should save status" do
+        @pipeline.save!
+        lambda {@pipeline.perform}.should raise_error(RecoverableError)
+        @pipeline.reload.status.should == :in_progress
+      end
+    end
+
+    describe "- execution (recoverable error that requires user input)" do
+      before(:each) do
+        failed_stage = SecondStage.new
+        failed_stage.stub!(:run).and_raise(RecoverableError.new('message', true))
+        SecondStage.stub!(:new).and_return(failed_stage)
+        @pipeline = SamplePipeline.new
+      end
+
+      it "should not re-raise error" do
+        lambda {@pipeline.perform}.should_not raise_error(RecoverableError)
+      end
+      
+      it "should update status" do
+        @pipeline.perform
+        @pipeline.status.should == :paused
+      end
+      
+      it "should save status" do
+        @pipeline.save!
+        @pipeline.perform
+        @pipeline.reload.status.should == :paused
+      end
+    end
+
+    describe "- execution (other errors will pause the pipeline)" do
+      before(:each) do
+        failed_stage = SecondStage.new
+        failed_stage.stub!(:run).and_raise(StandardError.new)
+        SecondStage.stub!(:new).and_return(failed_stage)
+        @pipeline = SamplePipeline.new
+      end
+
+      it "should not re-raise error" do
+        lambda {@pipeline.perform}.should_not raise_error(StandardError)
+      end
+      
+      it "should update status" do
+        @pipeline.perform
+        @pipeline.status.should == :paused
+      end
+      
+      it "should save status" do
+        @pipeline.save!
+        @pipeline.perform
+        @pipeline.reload.status.should == :paused
+      end
+    end
+
+    describe "- execution (retrying)" do
+      before(:each) do
+        @passing_stage = FirstStage.new
+        FirstStage.stub!(:new).and_return(@passing_stage)
+        
+        @failed_stage = SecondStage.new
+        @failed_stage.stub!(:run).and_raise(RecoverableError.new('message', true))
+        SecondStage.stub!(:new).and_return(@failed_stage)
+        @pipeline = SamplePipeline.new
+      end
+
+      it "should not re-raise error" do
+        lambda {@pipeline.perform}.should_not raise_error(RecoverableError)
+      end
+      
+      it "should update status" do
+        @pipeline.perform
+        @pipeline.status.should == :paused
+      end
+      
+      it "should save status" do
+        @pipeline.save!
+        @pipeline.perform
+        @pipeline.reload.status.should == :paused
+      end
+      
+      it "should skip completed stages" do
+        @pipeline.perform
+        @passing_stage.attempts.should == 1
+        @failed_stage.attempts.should == 1
+        
+        @pipeline.perform
+        @passing_stage.attempts.should == 1
+        @failed_stage.attempts.should == 2
+      end
+    end
+    
+    describe "- execution (state transitions)" do
+      before(:each) do
+        @pipeline = Base.new
+      end
+      
+      it "should execute if status is :not_started" do
+        @pipeline.should be_ok_to_resume
+        lambda {@pipeline.perform}.should_not raise_error(InvalidStatusError)
+      end
+
+      it "should execute if status is :paused (for retrying)" do
+        @pipeline.send(:status=, :paused)
+        
+        @pipeline.should be_ok_to_resume
+        lambda {@pipeline.perform}.should_not raise_error(InvalidStatusError)
+      end
+      
+      it "should not execute if status is :in_progress" do
+        @pipeline.send(:status=, :in_progress)
+        
+        @pipeline.should_not be_ok_to_resume
+        lambda {@pipeline.perform}.should raise_error(InvalidStatusError, "Status is already in progress")
+      end
+
+      it "should not execute if status is :completed" do
+        @pipeline.send(:status=, :completed)
+        
+        @pipeline.should_not be_ok_to_resume
+        lambda {@pipeline.perform}.should raise_error(InvalidStatusError, "Status is already completed")
+      end
+
+      it "should not execute if status is :failed" do
+        @pipeline.send(:status=, :failed)
+        
+        @pipeline.should_not be_ok_to_resume
+        lambda {@pipeline.perform}.should raise_error(InvalidStatusError, "Status is already failed")
+      end
+    end
+    
+    describe "- cancelling" do
+      before(:each) do
+        @passing_stage = FirstStage.new
+        FirstStage.stub!(:new).and_return(@passing_stage)
+        
+        @failed_stage = SecondStage.new
+        @failed_stage.stub!(:run).and_raise(RecoverableError.new('message', true))
+        SecondStage.stub!(:new).and_return(@failed_stage)
+        @pipeline = SamplePipeline.new
+        @pipeline.perform
+      end
+
+      it "should update status" do
+        @pipeline.cancel
+        @pipeline.status.should == :failed
+      end
+      
+      it "should save status" do
+        @pipeline.save!
+        @pipeline.cancel
+        @pipeline.reload.status.should == :failed
+      end
+    end
+
+    describe "- cancelling (state transitions)" do
+      before(:each) do
+        @pipeline = Base.new
+      end
+      
+      it "should cancel if status is :not_started" do
+        lambda {@pipeline.cancel}.should_not raise_error(InvalidStatusError)
+      end
+
+      it "should cancel if status is :paused (for retrying)" do
+        @pipeline.send(:status=, :paused)
+        
+        lambda {@pipeline.cancel}.should_not raise_error(InvalidStatusError)
+      end
+      
+      it "should not cancel if status is :in_progress" do
+        @pipeline.send(:status=, :in_progress)
+        
+        lambda {@pipeline.cancel}.should raise_error(InvalidStatusError, "Status is already in progress")
+      end
+
+      it "should not cancel if status is :completed" do
+        @pipeline.send(:status=, :completed)
+        
+        lambda {@pipeline.cancel}.should raise_error(InvalidStatusError, "Status is already completed")
+      end
+
+      it "should not cancel if status is :failed" do
+        @pipeline.send(:status=, :failed)
+        
+        lambda {@pipeline.cancel}.should raise_error(InvalidStatusError, "Status is already failed")
+      end
+    end
+    
+  end
+end

data/spec/pipeline/core_ext/symbol_attribute_spec.rb

@@ -0,0 +1,38 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+# Reusing stage table to simplify database integration tests
+class FakeForSymbolAttribute < ActiveRecord::Base
+  set_table_name :pipeline_stages
+  
+  symbol_attr :status
+end
+
+module Pipeline
+  describe SymbolAttribute do
+    before(:each) do
+      FakeForSymbolAttribute.delete_all
+    end
+    
+    it "should extend active record to allow symbol attributes to be saved as string" do
+      obj = FakeForSymbolAttribute.new(:status => 'started')
+      obj.save!
+      obj.status.should == :started
+      obj.reload.status.should == :started
+      
+      obj.status = 'finished'
+      obj.save!
+      obj.status.should == :finished
+      obj.reload.status.should == :finished
+    end
+
+    it "should extend Symbol to allow symbol attributes in conditions" do
+      objs = FakeForSymbolAttribute.find(:all, :conditions => ['status = ?', :started])
+      objs.should be_empty
+      
+      FakeForSymbolAttribute.create(:status => :started)
+
+      objs = FakeForSymbolAttribute.find(:all, :conditions => ['status = ?', :started])
+      objs.size.should == 1
+    end
+  end
+end

data/spec/pipeline/core_ext/transactional_attribute_spec.rb

@@ -0,0 +1,22 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+# Reusing stage table to simplify database integration tests
+class FakeForTransactionalAttribute < ActiveRecord::Base
+  set_table_name :pipeline_stages
+  
+  transactional_attr :status
+end
+
+module Pipeline
+  describe TransactionalAttribute do
+    it "should extend active record to allow transactional attributes to be saved in a nested transaction" do
+      obj = FakeForTransactionalAttribute.create(:status => "started")
+      obj.status.should == "started"
+      obj.reload.status.should == "started"
+      
+      obj.status = "finished"
+      obj.status.should == "finished"
+      obj.reload.status.should == "finished"
+    end
+  end
+end