pipeline 0.0.8 → 0.0.9

data/.gitignore

@@ -1,7 +1,8 @@
 *.sw?
 .DS_Store
 coverage
+doc
 rdoc
 pkg
 pipeline.log
-pipeline.sqlite
+pipeline.sqlite

data/CHANGELOG

@@ -1,3 +1,17 @@
+0.0.9
+=====
+
+Bug Fix:
+* Syntax error on generated migration
+
+Features:
+* Upgrading dependency to delayed_job
+* Move to gemcutter
+* Can be installed as a plugin or gem
+* Improved Documentation
+* Callbacks before and after stage/pipeline execution
+* Association to an external entity through external_id (Please refer to documentation)
+
 0.0.8
 =====
 

data/README.rdoc

@@ -4,6 +4,10 @@
 
 Pipeline is a Rails plugin/gem to run asynchronous processes in a configurable pipeline.
 
+== Documentation
+
+http://rdoc.info/projects/dtsato/pipeline
+
 == Features
 
 * Execution of sequential user-defined stages in an asynchronous pipeline
@@ -12,36 +16,54 @@ Pipeline is a Rails plugin/gem to run asynchronous processes in a configurable p
   * Irrecoverable errors fail the entire pipeline
   * Recoverable errors are automatically retried (using dj's exponential retry strategy)
   * Recoverable errors that require user input pause the pipeline for further retry
-* Cancelling of a paused pipeline
+* Cancelling/Resuming of a paused pipeline
+* Callbacks before and after executing stages and pipeline
+
+== Installation and Use
 
-Known Issues:
-* Rails' tests/specs will only work if delayed_job is installed as a gem, rather than a plugin.
++pipeline+ can be installed as either a RubyGem (recommended) or as a plugin.
 
-== Installation
+=== Gem
 
-Add the following lines to your config/environment.rb file:
+To install +pipeline+ as a RubyGem, add the following lines to your
+<tt>config/environment.rb</tt> file:
 
   config.gem "pipeline"
+  config.gem "delayed_job"
 
-Run the following:
+And execute:
 
   rake gems:install
   rake gems:unpack # Optional, if you want to vendor the gem
+
+=== Plugin
+
+To install it as a plugin, run:
+
+  script/plugin install git://github.com/dtsato/pipeline.git
+
+=== Generating the required tables
+
+In order to persist your pipelines and stages, execute the following:
+
   script/generate pipeline # To generate the migration scripts that will store pipelines
+  script/generate delayed_job # To generate the migration scripts for delayed_job
   rake db:migrate
 
+=== Starting your workers
+
 You will also need to run your Delayed Job workers that will process the pipeline jobs in the background:
 
   rake jobs:work
 
 == Dependencies
 
-* Rails
-* Delayed job (http://github.com/collectiveidea/delayed_job/tree/master)
+* Rails (ActiveRecord)
+* Delayed job (http://github.com/collectiveidea/delayed_job)
 
 == Usage
 
-Check <tt>examples</tt> for more examples (including error-recovery and cancelling)
+Check +examples+ for more examples (including error-recovery and cancelling)
 
   class Step1 < Pipeline::Stage::Base
     def perform

data/Rakefile

@@ -18,10 +18,12 @@ begin
     gem.test_files = Dir['spec/**/*'] + Dir['spec/*']
     
     gem.add_dependency('activerecord', '>= 2.0')
-    gem.add_dependency('collectiveidea-delayed_job', '>= 1.8.0')
+    gem.add_dependency('delayed_job', '>= 1.8.0')
     
     gem.rubyforge_project = "pipeline"
   end
+  
+  Jeweler::GemcutterTasks.new
 
 rescue LoadError
   puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"

data/TODO

@@ -1,7 +1,6 @@
-* Allow delayed_job to be used as a plugin, as well as a gem
 * Add examples of view display 
 * view helpers?
-* Improve documentation (RDoc)
+* Chaining stages with string (to avoid having to load stages before pipeline)
 * Adapter for other persistence frameworks (ActiveModel?)
 * Adapter for other background processing mechanisms (currently relying on dj's auto-retry)
 * Max attempts on pipeline (or stage)

data/VERSION

@@ -1 +1 @@
-0.0.8
+0.0.9

data/generators/pipeline/templates/migration.rb

@@ -4,6 +4,8 @@ class CreatePipelineInstancesAndStages < ActiveRecord::Migration
       t.string  :type                         # For single table inheritance
       t.string  :status                       # Current status of the pipeline
       t.integer :attempts, :default => 0      # Number of times this pipeline was executed
+      t.references :external                  # External object, to which this pipeline
+                                              # is associated (user-defined)
 
       t.timestamps
     end
@@ -17,6 +19,7 @@ class CreatePipelineInstancesAndStages < ActiveRecord::Migration
       t.integer    :attempts, :default => 0       # Number of times this stage was executed
 
       t.timestamps
+    end
   end
   
   def self.down

data/lib/pipeline.rb

@@ -1,4 +1,4 @@
-autoload :ActiveRecord, 'activerecord'
+autoload :ActiveRecord, 'active_record'
 autoload :Delayed, 'delayed_job'
 
 $: << File.dirname(__FILE__)
@@ -9,6 +9,7 @@ require 'pipeline/base'
 require 'pipeline/errors'
 require 'pipeline/stage/base'
 
+# Please refer to Pipeline::Base and Pipeline::Stage::Base for detailed documentation
 module Pipeline
   extend(ApiMethods)
 end

data/lib/pipeline/api_methods.rb

@@ -1,5 +1,11 @@
 module Pipeline
+  # This is the external API for Pipeline. Its methods should be called by client
+  # code wanting to manipulate/execute pipelines.
   module ApiMethods
+    
+    # Used to enqueue a pipeline execution. Raises InvalidPipelineError if the passed
+    # in argument is not a subclass of Pipeline::Base. The pipeline will be saved (if
+    # not already) and its <tt>id</tt> will be returned.
     def start(pipeline)
       raise InvalidPipelineError.new("Invalid pipeline") unless pipeline.is_a?(Pipeline::Base)
       pipeline.save! if pipeline.new_record?
@@ -7,14 +13,22 @@ module Pipeline
       pipeline.id
     end
     
+    # Enqueues execution of a paused pipeline for retrying. Raises InvalidPipelineError
+    # if a pipeline can not be found with the provided <tt>id</tt>. Raises
+    # InvalidStatusError if pipeline is in an invalid state for resuming (e.g. already
+    # cancelled, or completed)
     def resume(id)
       pipeline = Base.find(id)
-      raise InvalidStatusError.new(pipeline.status) unless pipeline.ok_to_resume?
+      pipeline.resume
       Delayed::Job.enqueue(pipeline)
     rescue ActiveRecord::RecordNotFound
       raise InvalidPipelineError.new("Invalid pipeline")
     end
     
+    # Cancels execution of a paused pipeline. Raises InvalidPipelineError if a pipeline
+    # can not be found with the provided <tt>id</tt>. Raises InvalidStatusError if
+    # pipeline is in an invalid state for cancelling (e.g. already cancelled, or
+    # completed)
     def cancel(id)
       pipeline = Base.find(id)
       pipeline.cancel

data/lib/pipeline/base.rb

@@ -1,16 +1,163 @@
 module Pipeline
+  # == Pipeline Stages
+  #
+  # Each pipeline is composed of sequential stages (see Pipeline::Stage::Base).
+  # The stages that will be executed are defined as follows:
+  #
+  #   class PrepareIngredients < Pipeline::Stage::Base
+  #     def run
+  #       puts "Slicing..."
+  #     end
+  #   end
+  #
+  #   class Cook < Pipeline::Stage::Base
+  #     def run
+  #       puts "Cooking..."
+  #     end
+  #   end
+  #
+  #   class MakeDinnerPipeline < Pipeline::Base
+  #     define_stages PrepareIngredients >> Cook
+  #   end
+  #
+  # When this pipeline executes, it will run each stage sequentially, and the output
+  # would be:
+  #   Slicing...
+  #   Cooking...
+  #
+  # A pipeline can get access to its stages through the <tt>stages</tt> association.
+  #
+  # == Error Handling
+  #
+  # There are 3 types of errors that a failed stage can specifically raise:
+  #
+  # * <b>Recoverable (requires user action)</b>: If a stage raises RecoverableError with
+  #   <tt>input_required? == true</tt>, the pipeline gets :paused and can be
+  #   resumed or cancelled by calling #resume and #cancel, respectively.
+  # 
+  # * <b>Recoverable (can be automatically retried)</b>: If a stage raises
+  #   RecoverableError with <tt>input_required? == false</tt>, the pipeline goes into
+  #   :retry state and will be automatically retried. This is currently achieved by
+  #   +delayed_job+'s retry mechanism. Please refer to 
+  #   http://github.com/collectiveidea/delayed_job for information about how to
+  #   configure the maximum number of retry attempts.
+  #
+  # * <b>Irrecoverable</b>: If a stage fails with an IrrecoverableError, the pipeline
+  #   gets :failed and therefore cannot be resumed or restarted.
+  #
+  # If a stage fails with any other type of error, you can choose the default behaviour
+  # for what happens to the pipeline. By default, the pipeline will pause, so it can be
+  # later resumed. This can be overriden by calling +default_failure_mode+ like:
+  #
+  #  class SamplePipeline < Pipeline::Base
+  #    self.default_failure_mode = :cancel
+  #  end
+  # 
+  # You can always go back to the default mode by calling:
+  #   self.default_failure_mode = :pause
+  #
+  # == State Transitions
+  # 
+  # The following diagram represents the state transitions a pipeline instance can
+  # go through during its life-cycle:
+  #
+  # :not_started ---> :in_progress ---> :completed / :failed
+  #                       ^ |
+  #                       | v
+  #                 :paused / :retry
+  #
+  # [:not_started] The pipeline was instantiated but not started yet.
+  # [:in_progress] After started or resumed, the pipeline remains on this state while
+  #                the stages are running.
+  # [:paused]      If a stage fails with a recoverable error that requires user action,
+  #                the pipeline gets paused.
+  # [:retry]       If a stage fails with a recoverable error that can be automatically
+  #                retried, the pipeline goes into this stage.
+  # [:completed]   After successfully running all stages, the pipeline is completed.
+  # [:failed]      If a stage fails with an unrecoverable error, or if the pipeline is
+  #                cancelled, it goes into this stage.
+  #
+  # == Referencing External Objects
+  #
+  # The execution of a pipeline will usually be associated to an external entity
+  # (e.g. a +User+ if the stages represent an internal user registration process, or a
+  # +Recipe+ in the examples of this page). To be able to reference the associated object
+  # from the stages, Pipeline::Base has an attribute <tt>external_id</tt> that can be
+  # used on a custom association to any external entity. Example:
+  #
+  #   class MakeDinnerPipeline < Pipeline::Base
+  #     define_stages PrepareIngredients >> Cook
+  #     belongs_to :recipe, :foreign_key => 'external_id'
+  #   end
+  #
+  # A Stage can reference this object as such:
+  #
+  #   class Cook < Pipeline::Stage::Base
+  #     def run
+  #       puts "Cooking a delicious #{pipeline.recipe.name}"
+  #     end
+  #   end
+  #
+  # == Callbacks
+  # 
+  # You can define custom callbacks to be called before (+before_pipeline+) and after
+  # (+after_pipeline+) executing a pipeline. Example:
+  #
+  #   class PrepareIngredients < Pipeline::Stage::Base
+  #     def run
+  #       puts "Slicing..."
+  #     end
+  #   end
+  #
+  #   class Cook < Pipeline::Stage::Base
+  #     def run
+  #       puts "Cooking..."
+  #     end
+  #   end
+  #
+  #   class MakeDinnerPipeline < Pipeline::Base
+  #     define_stages PrepareIngredients >> Cook
+  #
+  #     before_pipeline :wash_hands
+  #     after_pipeline :serve_dinner
+  #     
+  #     private
+  #     def wash_hands
+  #       puts "Washing hands before we start..."
+  #     end
+  #
+  #     def serve_dinner
+  #       puts "bon appetit!"
+  #     end
+  #   end
+  # 
+  #   Pipeline.start(MakeDinnerPipeline.new)
+  #
+  # Outputs:
+  #   Washing hands before we start...
+  #   Slicing...
+  #   Cooking...
+  #   bon appetit!
+  #
+  # Callbacks can be defined as a symbol that calls a private/protected method (like the
+  # example above), as an inline block, or as a +Callback+ object, as a regular
+  # +ActiveRecord+ callback.
   class Base < ActiveRecord::Base
     set_table_name :pipeline_instances
     
-    # :not_started ---> :in_progress ---> :completed
-    #                       ^ |       \-> :failed
+    # :not_started ---> :in_progress ---> :completed / :failed
+    #                       ^ |
     #                       | v
     #                 :paused / :retry
     symbol_attr :status
     transactional_attr :status
     private :status=
 
-    has_many :stages, :class_name => 'Pipeline::Stage::Base', :foreign_key => 'pipeline_instance_id', :dependent => :destroy
+    # Allows access to the associated stages
+    has_many :stages,
+      :class_name => 'Pipeline::Stage::Base',
+      :foreign_key => 'pipeline_instance_id',
+      :dependent => :destroy
 
     class_inheritable_accessor :defined_stages, :instance_writer => false
     self.defined_stages = []
@@ -18,15 +165,33 @@ module Pipeline
     class_inheritable_accessor :failure_mode, :instance_writer => false
     self.failure_mode = :pause
     
+    define_callbacks :before_pipeline, :after_pipeline
+    
+    # Defines the stages of this pipeline. Please refer to section
+    # <em>"Pipeline Stages"</em> above
     def self.define_stages(stages)
       self.defined_stages = stages.build_chain
     end
 
+    # Sets the behaviour of this pipeline when a failure occurs. Accepted symbols are:
+    #
+    # [:pause]  Pauses the pipeline on failure (default)
+    # [:cancel] Fails the pipeline on failure
     def self.default_failure_mode=(mode)
       new_mode = [:pause, :cancel].include?(mode) ? mode : :pause
       self.failure_mode = new_mode
     end
 
+    # Standard ActiveRecord callback to setup initial stages and status
+    # when a new pipeline is instantiated. If you override this callback, make
+    # sure to call +super+:
+    #
+    #   class SamplePipeline < Pipeline::Base
+    #     def after_initialize
+    #       super
+    #       self[:special_attribute] ||= "standard value"
+    #     end
+    #   end
     def after_initialize
       if new_record?
         self[:status] = :not_started
@@ -36,42 +201,68 @@ module Pipeline
       end
     end
     
+    # Standard +delayed_job+ method called when executing this pipeline. Raises   
+    # InvalidStatusError if pipeline is in an invalid state for execution (e.g.
+    # already cancelled, or completed).
+    #
+    # This method will be called by +delayed_job+
+    # if this object is enqueued for asynchronous execution. However, you could
+    # call this method and execute the pipeline synchronously, without relying on
+    # +delayed_job+. Auto-retry would not work in this case, though.
     def perform
-      reload unless new_record?
-      raise InvalidStatusError.new(status) unless ok_to_resume?
+      _check_valid_status
       begin
         _setup
         stages.each do |stage|
           stage.perform unless stage.completed?
         end
-        self.status = :completed
+        _complete_with_status(:completed)
       rescue IrrecoverableError
-        self.status = :failed
+        _complete_with_status(:failed)
       rescue RecoverableError => e
         if e.input_required?
-          self.status = :paused
+          _complete_with_status(:paused)
         else
-          self.status = :retry
+          _complete_with_status(:retry)
           raise e
         end
       rescue Exception
-        self.status = (failure_mode == :cancel ? :failed : :paused)
+        _complete_with_status(failure_mode == :cancel ? :failed : :paused)
       end
     end
     
+    # Attempts to cancel this pipeline. Raises InvalidStatusError if pipeline is in
+    # an invalid state for cancelling (e.g. already cancelled, or completed)
     def cancel
-      raise InvalidStatusError.new(status) unless ok_to_resume?
-      self.status = :failed
+      _check_valid_status
+      _complete_with_status(:failed)
     end
     
+    # Attempts to resume this pipeline. Raises InvalidStatusError if pipeline is in
+    # an invalid state for resuming (e.g. already cancelled, or completed)
+    def resume
+      _check_valid_status
+    end
+    
+    private
     def ok_to_resume?
       [:not_started, :paused, :retry].include?(status)
     end
 
-    private
+    def _check_valid_status
+      reload unless new_record?
+      raise InvalidStatusError.new(status) unless ok_to_resume?
+    end
+    
     def _setup
       self.attempts += 1
       self.status = :in_progress
+      run_callbacks(:before_pipeline)
+    end
+    
+    def _complete_with_status(status)
+      self.status = status
+      run_callbacks(:after_pipeline)
     end
   end
 end