pipeline 0.0.1

data/.gitignore

@@ -0,0 +1,7 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg
+pipeline.log
+pipeline.sqlite

data/CHANGELOG

@@ -0,0 +1,12 @@
+0.0.1
+=====
+
+Initial Release:
+* Execution of sequential user-defined stages in an asynchronous pipeline
+* Persistence of pipeline instances and stages
+* Error recovery strategies:
+  * Irrecoverable errors fail the entire pipeline
+  * Recoverable errors are automatically retried (using dj's exponential retry strategy)
+  * Recoverable errors that require user input pause the pipeline for further retry
+* Cancelling of a paused pipeline
+

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 dtsato.com, Danilo Sato
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,67 @@
+= Pipeline
+
+== Description
+
+Pipeline is a Rails plugin/gem to run asynchronous processes in a configurable pipeline.
+
+== Features
+
+* Execution of sequential user-defined stages in an asynchronous pipeline
+* Persistence of pipeline instances and stages
+* Error recovery strategies:
+  * Irrecoverable errors fail the entire pipeline
+  * Recoverable errors are automatically retried (using dj's exponential retry strategy)
+  * Recoverable errors that require user input pause the pipeline for further retry
+* Cancelling of a paused pipeline
+
+== Installation
+
+Add the following lines to your config/environment.rb file:
+
+  config.gem "dtsato-pipeline", :version => ">= 0.0.1", :source => "http://gems.github.com"
+
+Run the following:
+
+  rake gems:install
+  rake gems:unpack # Optional, if you want to vendor the gem
+  script/generate pipeline # To generate the migration scripts that will store pipelines
+  rake db:migrate
+
+You will also need to run your Delayed Job workers that will process the pipeline jobs in the background:
+
+  rake jobs:work
+
+== Dependencies
+
+* Rails
+* Delayed job (http://github.com/collectiveidea/delayed_job/tree/master)
+
+== Usage
+
+Check <tt>examples</tt> for more examples (including error-recovery and cancelling)
+
+  class Step1 < Pipeline::Stage::Base
+    def perform
+      puts("Started step 1")
+      sleep 2
+      puts("Finished step 1")
+    end
+  end
+
+  class Step2 < Pipeline::Stage::Base
+    def perform
+      puts("Started step 2")
+      sleep 3
+      puts("Finished step 2")
+    end
+  end
+  
+  class TwoStepPipeline < Pipeline::Base
+    define_stages Step1 >> Step2
+  end
+
+  Pipeline.start(TwoStepPipeline.new)
+
+== Copyright
+
+Copyright (c) 2009 Danilo Sato. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,104 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "pipeline"
+    gem.summary  = "A Rails plugin/gem to run asynchronous processes in a configurable pipeline"
+    gem.email = "danilo@dtsato.com"
+    gem.homepage = "http://github.com/dtsato/pipeline"
+    gem.authors = ["Danilo Sato"]
+    gem.description = "Pipeline is a Rails plugin/gem to run asynchronous processes in a configurable pipeline."
+
+    gem.has_rdoc = true
+    gem.rdoc_options = ["--main", "README.rdoc", "--inline-source", "--line-numbers"]
+    gem.extra_rdoc_files = ["README.rdoc"]
+
+    gem.test_files = Dir['spec/**/*']
+    
+    gem.add_dependency('activerecord', '>= 2.0')
+    gem.add_dependency('collectiveidea-delayed_job', '>= 1.8.0')
+    
+    gem.rubyforge_project = "pipeline"
+  end
+
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+# These are new tasks
+begin
+  require 'rake/contrib/sshpublisher'
+  namespace :rubyforge do
+
+    desc "Release gem and RDoc documentation to RubyForge"
+    task :release => ["rubyforge:release:gem", "rubyforge:release:docs"]
+
+    namespace :release do
+      desc "Publish RDoc to RubyForge."
+      task :docs => [:rdoc] do
+        config = YAML.load(
+            File.read(File.expand_path('~/.rubyforge/user-config.yml'))
+        )
+
+        host = "#{config['username']}@rubyforge.org"
+        remote_dir = "/var/www/gforge-projects/pipeline/"
+        local_dir = 'rdoc'
+
+        Rake::SshDirPublisher.new(host, remote_dir, local_dir).upload
+      end
+    end
+  end
+rescue LoadError
+  puts "Rake SshDirPublisher is unavailable or your rubyforge environment is not configured."
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+  spec.spec_opts = ['--options', "\"spec/spec.opts\""]
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov_opts = lambda do
+    IO.readlines("spec/rcov.opts").map {|l| l.chomp.split " "}.flatten
+  end
+  spec.rcov = true
+end
+
+begin
+  require "synthesis/task"
+
+  desc "Run Synthesis on specs"
+  Synthesis::Task.new("spec:synthesis") do |t|
+    t.adapter = :rspec
+    t.pattern = 'spec/**/*_spec.rb'
+    t.ignored = ['Pipeline::FakePipeline', 'Delayed::Job']
+  end
+rescue LoadError
+  desc 'Synthesis rake task not available'
+  task "spec:synthesis" do
+    abort 'Synthesis rake task is not available. Be sure to install synthesis as a gem'
+  end
+end
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION.yml')
+    config = YAML.load(File.read('VERSION.yml'))
+    version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "pipeline #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+task :default => :spec

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/examples/auto_recoverable_pipeline.rb

@@ -0,0 +1,35 @@
+require File.join(File.dirname(__FILE__), '..', 'init')
+require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
+ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+class Step1 < Pipeline::Stage::Base
+  def run
+    puts("Started step 1")
+    # Will fail on the first time, but pass on the second
+    if attempts == 1
+      puts("Raising auto-recoverable error")
+      raise Pipeline::RecoverableError.new
+    end
+    puts("Finished step 1")
+  end
+end
+
+class Step2 < Pipeline::Stage::Base
+  def run
+    puts("Started step 2")
+    # Will fail on the first time, but pass on the second
+    if attempts == 1
+      puts("Raising another auto-recoverable error")
+      raise Pipeline::RecoverableError.new
+    end
+    puts("Finished step 2")
+  end
+end
+
+class TwoStepPipeline < Pipeline::Base
+  define_stages Step1 >> Step2
+end
+
+Pipeline.start(TwoStepPipeline.new)
+
+Delayed::Worker.new.start

data/examples/cancelling_pipeline.rb

@@ -0,0 +1,32 @@
+require File.join(File.dirname(__FILE__), '..', 'init')
+require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
+ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+class Step1 < Pipeline::Stage::Base
+  def run
+    puts("Started step 1")
+    puts("Raising user-recoverable error")
+    raise Pipeline::RecoverableError.new("require your action", true)
+  end
+end
+
+class Step2 < Pipeline::Stage::Base
+  def run
+    puts("Started step 2")
+    sleep 3
+    puts("Finished step 2")
+  end
+end
+
+class TwoStepPipeline < Pipeline::Base
+  define_stages Step1 >> Step2
+end
+
+id = Pipeline.start(TwoStepPipeline.new)
+
+Delayed::Worker.new.start
+
+# CTRL-C to execute the cancelling, since we want to cancel after the stage failed, but
+# Worker is blocking the process on the previous line
+Pipeline.cancel(id)
+p Pipeline::Base.find(id)

data/examples/two_step_pipeline.rb

@@ -0,0 +1,27 @@
+require File.join(File.dirname(__FILE__), '..', 'init')
+require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
+ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+class Step1 < Pipeline::Stage::Base
+  def run
+    puts("Started step 1")
+    sleep 2
+    puts("Finished step 1")
+  end
+end
+
+class Step2 < Pipeline::Stage::Base
+  def run
+    puts("Started step 2")
+    sleep 3
+    puts("Finished step 2")
+  end
+end
+
+class TwoStepPipeline < Pipeline::Base
+  define_stages Step1 >> Step2
+end
+
+Pipeline.start(TwoStepPipeline.new)
+
+Delayed::Worker.new.start

data/examples/user_recoverable_pipeline.rb

@@ -0,0 +1,33 @@
+require File.join(File.dirname(__FILE__), '..', 'init')
+require File.join(File.dirname(__FILE__), '..', 'spec', 'database_integration_helper')
+ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+class Step1 < Pipeline::Stage::Base
+  def run
+    puts("Started step 1")
+    sleep 2
+    puts("Finished step 1")
+  end
+end
+
+class Step2 < Pipeline::Stage::Base
+  def run
+    puts("Started step 2")
+    # Will fail on the first time, but pass on the second
+    if attempts == 1
+      puts("Raising user-recoverable error")
+      raise Pipeline::RecoverableError.new("require your action", true)
+    end
+    puts("Finished step 2")
+  end
+end
+
+class TwoStepPipeline < Pipeline::Base
+  define_stages Step1 >> Step2
+end
+
+id = Pipeline.start(TwoStepPipeline.new)
+
+Pipeline.resume(id)
+
+Delayed::Worker.new.start

data/generators/pipeline/pipeline_generator.rb

@@ -0,0 +1,12 @@
+class PipelineGenerator < Rails::Generator::Base
+  
+  def manifest
+    record do |m|
+      m.migration_template "pipeline_instances_migration.rb", 'db/migrate',
+                           :migration_file_name => "create_pipeline_instances"
+      m.migration_template "pipeline_stages_migration.rb", 'db/migrate',
+                          :migration_file_name => "create_pipeline_stages"
+    end
+  end
+  
+end

data/generators/pipeline/templates/pipeline_instances_migration.rb

@@ -0,0 +1,16 @@
+class CreatePipelineInstances < ActiveRecord::Migration
+  def self.up
+    create_table :pipeline_instances, :force => true do |t|
+      t.string  :type                         # For single table inheritance
+      t.string  :status                       # Current status of the pipeline
+      t.integer :attempts, :default => 0      # Number of times this pipeline was executed
+
+      t.timestamps
+    end
+
+  end
+  
+  def self.down
+    drop_table :pipeline_instances  
+  end
+end

data/generators/pipeline/templates/pipeline_stages_migration.rb

@@ -0,0 +1,19 @@
+class CreatePipelineStages < ActiveRecord::Migration
+  def self.up
+    create_table :pipeline_stages, :force => true do |t|
+      t.references :pipeline_instance             # Pipeline that holds this stage
+      t.string     :type                          # For single table inheritance
+      t.string     :name                          # Name of the stage
+      t.string     :status                        # Current status of the stage
+      t.text       :message                       # Message that describes current status
+      t.integer    :attempts, :default => 0       # Number of times this stage was executed
+
+      t.timestamps
+    end
+
+  end
+  
+  def self.down
+    drop_table :pipeline_stages  
+  end
+end

data/init.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), 'lib', 'pipeline')

data/lib/pipeline.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+require 'activerecord'
+gem 'collectiveidea-delayed_job'
+autoload :Delayed, 'delayed_job'
+
+$: << File.dirname(__FILE__)
+require 'pipeline/core_ext/symbol_attribute'
+require 'pipeline/core_ext/transactional_attribute'
+require 'pipeline/api_methods'
+require 'pipeline/base'
+require 'pipeline/errors'
+require 'pipeline/stage/base'
+
+module Pipeline
+  extend(ApiMethods)
+end

data/lib/pipeline/api_methods.rb

@@ -0,0 +1,23 @@
+module Pipeline
+  module ApiMethods
+    def start(pipeline)
+      raise InvalidPipelineError.new("Invalid pipeline") unless pipeline.is_a?(Pipeline::Base)
+      pipeline.save!
+      Delayed::Job.enqueue(pipeline)
+      pipeline.id
+    end
+    
+    def resume(id)
+      pipeline = Base.find(id)
+      raise InvalidPipelineError.new("Invalid pipeline") unless pipeline
+      raise InvalidStatusError.new(pipeline.status) unless pipeline.ok_to_resume?
+      Delayed::Job.enqueue(pipeline)
+    end
+    
+    def cancel(id)
+      pipeline = Base.find(id)
+      raise InvalidPipelineError.new("Invalid pipeline") unless pipeline
+      pipeline.cancel
+    end
+  end
+end