dtsato-pipeline 0.0.1

data/.gitignore

@@ -0,0 +1,7 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg
+pipeline.log
+pipeline.sqlite

data/CHANGELOG

@@ -0,0 +1,12 @@
+0.0.1
+=====
+
+Initial Release:
+* Execution of sequential user-defined stages in an asynchronous pipeline
+* Persistence of pipeline instances and stages
+* Error recovery strategies:
+  * Irrecoverable errors fail the entire pipeline
+  * Recoverable errors are automatically retried (using dj's exponential retry strategy)
+  * Recoverable errors that require user input pause the pipeline for further retry
+* Cancelling of a paused pipeline
+

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 dtsato.com, Danilo Sato
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,67 @@
+= Pipeline
+
+== Description
+
+Pipeline is a Rails plugin/gem to run asynchronous processes in a configurable pipeline.
+
+== Features
+
+* Execution of sequential user-defined stages in an asynchronous pipeline
+* Persistence of pipeline instances and stages
+* Error recovery strategies:
+  * Irrecoverable errors fail the entire pipeline
+  * Recoverable errors are automatically retried (using dj's exponential retry strategy)
+  * Recoverable errors that require user input pause the pipeline for further retry
+* Cancelling of a paused pipeline
+
+== Installation
+
+Add the following lines to your config/environment.rb file:
+
+  config.gem "dtsato-pipeline", :version => ">= 0.0.1", :source => "http://gems.github.com"
+
+Run the following:
+
+  rake gems:install
+  rake gems:unpack # Optional, if you want to vendor the gem
+  script/generate pipeline # To generate the migration scripts that will store pipelines
+  rake db:migrate
+
+You will also need to run your Delayed Job workers that will process the pipeline jobs in the background:
+
+  rake jobs:work
+
+== Dependencies
+
+* Rails
+* Delayed job (http://github.com/collectiveidea/delayed_job/tree/master)
+
+== Usage
+
+Check <tt>examples</tt> for more examples (including error-recovery and cancelling)
+
+  class Step1 < Pipeline::Stage::Base
+    def perform
+      puts("Started step 1")
+      sleep 2
+      puts("Finished step 1")
+    end
+  end
+
+  class Step2 < Pipeline::Stage::Base
+    def perform
+      puts("Started step 2")
+      sleep 3
+      puts("Finished step 2")
+    end
+  end
+  
+  class TwoStepPipeline < Pipeline::Base
+    define_stages Step1 >> Step2
+  end
+
+  Pipeline.start(TwoStepPipeline.new)
+
+== Copyright
+
+Copyright (c) 2009 Danilo Sato. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,104 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "pipeline"
+    gem.summary  = "A Rails plugin/gem to run asynchronous processes in a configurable pipeline"
+    gem.email = "danilo@dtsato.com"
+    gem.homepage = "http://github.com/dtsato/pipeline"
+    gem.authors = ["Danilo Sato"]
+    gem.description = "Pipeline is a Rails plugin/gem to run asynchronous processes in a configurable pipeline."
+
+    gem.has_rdoc = true
+    gem.rdoc_options = ["--main", "README.rdoc", "--inline-source", "--line-numbers"]
+    gem.extra_rdoc_files = ["README.rdoc"]
+
+    gem.test_files = Dir['spec/**/*']
+    
+    gem.add_dependency('activerecord', '>= 2.0')
+    gem.add_dependency('collectiveidea-delayed_job', '>= 1.8.0')
+    
+    gem.rubyforge_project = "pipeline"
+  end
+
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+# These are new tasks
+begin
+  require 'rake/contrib/sshpublisher'
+  namespace :rubyforge do
+
+    desc "Release gem and RDoc documentation to RubyForge"
+    task :release => ["rubyforge:release:gem", "rubyforge:release:docs"]
+
+    namespace :release do
+      desc "Publish RDoc to RubyForge."
+      task :docs => [:rdoc] do
+        config = YAML.load(
+            File.read(File.expand_path('~/.rubyforge/user-config.yml'))
+        )
+
+        host = "#{config['username']}@rubyforge.org"
+        remote_dir = "/var/www/gforge-projects/pipeline/"
+        local_dir = 'rdoc'
+
+        Rake::SshDirPublisher.new(host, remote_dir, local_dir).upload
+      end
+    end
+  end
+rescue LoadError
+  puts "Rake SshDirPublisher is unavailable or your rubyforge environment is not configured."
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+  spec.spec_opts = ['--options', "\"spec/spec.opts\""]
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov_opts = lambda do
+    IO.readlines("spec/rcov.opts").map {|l| l.chomp.split " "}.flatten
+  end
+  spec.rcov = true
+end
+
+begin
+  require "synthesis/task"
+
+  desc "Run Synthesis on specs"
+  Synthesis::Task.new("spec:synthesis") do |t|
+    t.adapter = :rspec
+    t.pattern = 'spec/**/*_spec.rb'
+    t.ignored = ['Pipeline::FakePipeline', 'Delayed::Job']
+  end
+rescue LoadError
+  desc 'Synthesis rake task not available'
+  task "spec:synthesis" do
+    abort 'Synthesis rake task is not available. Be sure to install synthesis as a gem'
+  end
+end
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION.yml')
+    config = YAML.load(File.read('VERSION.yml'))
+    version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "pipeline #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+task :default => :spec

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/examples/auto_recoverable_pipeline.rb

@@ -0,0 +1,35 @@
+require File.join(File.dirname(__FILE__), '..', 'init')
+require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
+ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+class Step1 < Pipeline::Stage::Base
+  def run
+    puts("Started step 1")
+    # Will fail on the first time, but pass on the second
+    if attempts == 1
+      puts("Raising auto-recoverable error")
+      raise Pipeline::RecoverableError.new
+    end
+    puts("Finished step 1")
+  end
+end
+
+class Step2 < Pipeline::Stage::Base
+  def run
+    puts("Started step 2")
+    # Will fail on the first time, but pass on the second
+    if attempts == 1
+      puts("Raising another auto-recoverable error")
+      raise Pipeline::RecoverableError.new
+    end
+    puts("Finished step 2")
+  end
+end
+
+class TwoStepPipeline < Pipeline::Base
+  define_stages Step1 >> Step2
+end
+
+Pipeline.start(TwoStepPipeline.new)
+
+Delayed::Worker.new.start

data/examples/cancelling_pipeline.rb

@@ -0,0 +1,32 @@
+require File.join(File.dirname(__FILE__), '..', 'init')
+require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
+ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+class Step1 < Pipeline::Stage::Base
+  def run
+    puts("Started step 1")
+    puts("Raising user-recoverable error")
+    raise Pipeline::RecoverableError.new("require your action", true)
+  end
+end
+
+class Step2 < Pipeline::Stage::Base
+  def run
+    puts("Started step 2")
+    sleep 3
+    puts("Finished step 2")
+  end
+end
+
+class TwoStepPipeline < Pipeline::Base
+  define_stages Step1 >> Step2
+end
+
+id = Pipeline.start(TwoStepPipeline.new)
+
+Delayed::Worker.new.start
+
+# CTRL-C to execute the cancelling, since we want to cancel after the stage failed, but
+# Worker is blocking the process on the previous line
+Pipeline.cancel(id)
+p Pipeline::Base.find(id)

data/examples/two_step_pipeline.rb

@@ -0,0 +1,27 @@
+require File.join(File.dirname(__FILE__), '..', 'init')
+require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
+ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+class Step1 < Pipeline::Stage::Base
+  def run
+    puts("Started step 1")
+    sleep 2
+    puts("Finished step 1")
+  end
+end
+
+class Step2 < Pipeline::Stage::Base
+  def run
+    puts("Started step 2")
+    sleep 3
+    puts("Finished step 2")
+  end
+end
+
+class TwoStepPipeline < Pipeline::Base
+  define_stages Step1 >> Step2
+end
+
+Pipeline.start(TwoStepPipeline.new)
+
+Delayed::Worker.new.start

data/examples/user_recoverable_pipeline.rb

@@ -0,0 +1,33 @@
+require File.join(File.dirname(__FILE__), '..', 'init')
+require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
+ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+class Step1 < Pipeline::Stage::Base
+  def run
+    puts("Started step 1")
+    sleep 2
+    puts("Finished step 1")
+  end
+end
+
+class Step2 < Pipeline::Stage::Base
+  def run
+    puts("Started step 2")
+    # Will fail on the first time, but pass on the second
+    if attempts == 1
+      puts("Raising user-recoverable error")
+      raise Pipeline::RecoverableError.new("require your action", true)
+    end
+    puts("Finished step 2")
+  end
+end
+
+class TwoStepPipeline < Pipeline::Base
+  define_stages Step1 >> Step2
+end
+
+id = Pipeline.start(TwoStepPipeline.new)
+
+Pipeline.resume(id)
+
+Delayed::Worker.new.start

data/generators/pipeline/pipeline_generator.rb

@@ -0,0 +1,12 @@
+class PipelineGenerator < Rails::Generator::Base
+  
+  def manifest
+    record do |m|
+      m.migration_template "pipeline_instances_migration.rb", 'db/migrate',
+                           :migration_file_name => "create_pipeline_instances"
+      m.migration_template "pipeline_stages_migration.rb", 'db/migrate',
+                          :migration_file_name => "create_pipeline_stages"
+    end
+  end
+  
+end

data/generators/pipeline/templates/pipeline_instances_migration.rb

@@ -0,0 +1,16 @@
+class CreatePipelineInstances < ActiveRecord::Migration
+  def self.up
+    create_table :pipeline_instances, :force => true do |t|
+      t.string  :type                         # For single table inheritance
+      t.string  :status                       # Current status of the pipeline
+      t.integer :attempts, :default => 0      # Number of times this pipeline was executed
+
+      t.timestamps
+    end
+
+  end
+  
+  def self.down
+    drop_table :pipeline_instances  
+  end
+end

data/generators/pipeline/templates/pipeline_stages_migration.rb

@@ -0,0 +1,19 @@
+class CreatePipelineStages < ActiveRecord::Migration
+  def self.up
+    create_table :pipeline_stages, :force => true do |t|
+      t.references :pipeline_instance             # Pipeline that holds this stage
+      t.string     :type                          # For single table inheritance
+      t.string     :name                          # Name of the stage
+      t.string     :status                        # Current status of the stage
+      t.text       :message                       # Message that describes current status
+      t.integer    :attempts, :default => 0       # Number of times this stage was executed
+
+      t.timestamps
+    end
+
+  end
+  
+  def self.down
+    drop_table :pipeline_stages  
+  end
+end

data/init.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), 'lib', 'pipeline')

data/lib/pipeline/api_methods.rb

@@ -0,0 +1,23 @@
+module Pipeline
+  module ApiMethods
+    def start(pipeline)
+      raise InvalidPipelineError.new("Invalid pipeline") unless pipeline.is_a?(Pipeline::Base)
+      pipeline.save!
+      Delayed::Job.enqueue(pipeline)
+      pipeline.id
+    end
+    
+    def resume(id)
+      pipeline = Base.find(id)
+      raise InvalidPipelineError.new("Invalid pipeline") unless pipeline
+      raise InvalidStatusError.new(pipeline.status) unless pipeline.ok_to_resume?
+      Delayed::Job.enqueue(pipeline)
+    end
+    
+    def cancel(id)
+      pipeline = Base.find(id)
+      raise InvalidPipelineError.new("Invalid pipeline") unless pipeline
+      pipeline.cancel
+    end
+  end
+end

data/lib/pipeline/base.rb

@@ -0,0 +1,67 @@
+module Pipeline
+  class Base < ActiveRecord::Base
+    set_table_name :pipeline_instances
+    
+    # :not_started ---> :in_progress ---> :completed
+    #                       ^ |       \-> :failed
+    #                       | v
+    #                     :paused
+    symbol_attr :status
+    transactional_attr :status
+    private :status=
+
+    has_many :stages, :class_name => 'Pipeline::Stage::Base', :foreign_key => 'pipeline_instance_id', :dependent => :destroy
+
+    class_inheritable_accessor :defined_stages, :instance_writer => false
+    self.defined_stages = []
+    
+    def self.define_stages(stages)
+      self.defined_stages = stages.build_chain
+    end
+
+    def after_initialize
+      self[:status] = :not_started if new_record?
+      if new_record?
+        self.class.defined_stages.each do |stage_class|
+          stages << stage_class.new(:pipeline => self)
+        end
+      end
+    end
+    
+    def perform
+      raise InvalidStatusError.new(status) unless ok_to_resume?
+      begin
+        _setup
+        stages.each do |stage|
+          stage.perform unless stage.completed?
+        end
+        self.status = :completed
+      rescue IrrecoverableError
+        self.status = :failed
+      rescue RecoverableError => e
+        if e.input_required?
+          self.status = :paused
+        else
+          raise e
+        end
+      rescue
+        self.status = :paused
+      end
+    end
+    
+    def cancel
+      raise InvalidStatusError.new(status) unless ok_to_resume?
+      self.status = :failed
+    end
+    
+    def ok_to_resume?
+      [:not_started, :paused].include?(status)
+    end
+
+    private
+    def _setup
+      self.attempts += 1
+      self.status = :in_progress
+    end
+  end
+end

data/lib/pipeline/core_ext/symbol_attribute.rb

@@ -0,0 +1,29 @@
+module Pipeline
+  module SymbolAttribute
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      def symbol_attrs(*attributes)
+        attributes.each do |attribute|
+          class_eval <<-EOD
+            def #{attribute.to_s}
+              read_attribute('#{attribute.to_s}').to_sym rescue nil
+            end
+          EOD
+        end
+      end
+      
+      alias_method :symbol_attr, :symbol_attrs
+    end
+  end
+end
+
+class Symbol
+  def quoted_id
+    "'#{ActiveRecord::Base.connection.quote_string(self.to_s)}'"
+  end
+end
+
+ActiveRecord::Base.send(:include, Pipeline::SymbolAttribute)

data/lib/pipeline/core_ext/transactional_attribute.rb

@@ -0,0 +1,26 @@
+module Pipeline
+  module TransactionalAttribute
+    def self.included (base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      def transactional_attrs(*attributes)
+        attributes.each do |attribute|
+          class_eval <<-EOD
+            def #{attribute.to_s}=(value)
+              ActiveRecord::Base.transaction(:requires_new => true) do
+                write_attribute('#{attribute.to_s}', value)
+                save!
+              end
+            end
+          EOD
+        end
+      end
+      
+      alias_method :transactional_attr, :transactional_attrs
+    end
+  end
+end
+
+ActiveRecord::Base.send(:include, Pipeline::TransactionalAttribute)

data/lib/pipeline/errors.rb

@@ -0,0 +1,24 @@
+module Pipeline
+  class InvalidPipelineError < StandardError; end
+  
+  class InvalidStatusError < StandardError
+    def initialize(status)
+      super("Status is already #{status.to_s.gsub(/_/, ' ')}")
+    end
+  end
+
+  class IrrecoverableError < StandardError; end
+
+  class RecoverableError < StandardError
+    def initialize(msg = nil, input_required = false)
+      super(msg)
+      @input_required = input_required
+    end
+
+    def input_required?
+      @input_required
+    end
+  end
+
+  extend(ApiMethods)
+end

data/lib/pipeline/stage/base.rb

@@ -0,0 +1,60 @@
+module Pipeline
+  module Stage
+    class Base < ActiveRecord::Base
+      set_table_name :pipeline_stages
+      
+      # :not_started ---> :in_progress ---> :completed
+      #                       ^ |
+      #                       | v
+      #                     :failed
+      symbol_attr :status
+      transactional_attr :status
+      private :status=
+
+      belongs_to :pipeline, :class_name => "Pipeline::Base"
+      
+      @@chain = []
+      def self.>>(next_stage)
+        @@chain << self
+        next_stage
+      end
+      
+      def self.build_chain
+        chain = @@chain + [self]
+        @@chain = []
+        chain
+      end
+      
+      def after_initialize
+        self.name ||= self.class.to_s
+        self[:status] = :not_started if new_record?
+      end
+      
+      def completed?
+        status == :completed
+      end
+      
+      def perform
+        raise InvalidStatusError.new(status) unless [:not_started, :failed].include?(status)
+        begin
+          _setup
+          run
+          self.status = :completed
+        rescue => e
+          self.message = e.message
+          self.status = :failed
+          raise e
+        end
+      end
+      
+      # Subclass must implement this as part of the contract
+      def run; end
+      
+      private
+      def _setup
+        self.attempts += 1
+        self.status = :in_progress
+      end
+    end
+  end
+end

data/lib/pipeline.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+require 'activerecord'
+gem 'collectiveidea-delayed_job'
+autoload :Delayed, 'delayed_job'
+
+$: << File.dirname(__FILE__)
+require 'pipeline/core_ext/symbol_attribute'
+require 'pipeline/core_ext/transactional_attribute'
+require 'pipeline/api_methods'
+require 'pipeline/base'
+require 'pipeline/errors'
+require 'pipeline/stage/base'
+
+module Pipeline
+  extend(ApiMethods)
+end