resque-multi-step 1.0.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Peter Williams
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,117 @@
+resque-multi-step
+======
+
+Resque multi-step provides an abstraction for managing multiple step
+async tasks.
+
+Status
+----
+
+This software is not considered stable at this time.  Use at your own risk.
+
+Using multi-step tasks
+----
+
+Consider a situation where you need to perform several actions and
+would like those actions to run in parallel and you would like to keep
+track of the progress.  Mutli-step can be use to implement that as
+follows:
+
+    task = Resque::Plugins::MultiStepTask.create("pirate-take-over") do |task|
+      blog.posts.each do |post|                                     
+        task.add_job ConvertPostToPirateTalk, post.id
+      end
+    end
+    
+A resque job will be queued for each post.  The `task` object will
+keep track of how many of the tasks have been completed successfully
+(`#completed_count`).  That combined with the overall job count
+(`#total_job_count`) make it easy to compute the percentage completion
+of a mutli-step task.
+
+The failed job count (`#failed_count`) makes it easy to determine if
+problem has occurred during the execution.
+
+Looking up existing tasks
+----
+
+Once you have kicked off a job you can look it up again later using
+it's task id.  First you persist the task id when you create the task.
+
+    task = Resque::Plugins::MultiStepTask.create('pirate-take-over") do |task|
+      ...
+    end
+    blog.async_task_id = task.task_id
+    blog.save!
+
+Then you can look it up using the `.find` method on `MultiStepTask`.
+
+    # Progress reporting action; executed in a different process.
+    begin
+      task = Resque::Plugins::MultiStepTask.find(blog.async_task_id)
+      render :text => "percent complete #{(task.completed_count.quo(task.total_job_count) * 100).round}%
+      
+    rescue Resque::Plugins::MultiStepTask::NoSuchMultiStepTask
+      # task completed...
+     
+      redirect_to blog_url(blog)
+    end
+
+Finalization
+----
+
+Often when doing mutli-step tasks there are a bunch of tasks that can
+all happen in parallel and then a few that can only be executed after
+all the rest have completed.  Mutli-step task finalization supports
+just that use case.
+
+Using our example, say we want to commit the solr index and then
+unlock the blog we are converting to pirate talk once the conversion
+is complete.
+
+    task = Resque::Plugins::MultiStepTask.create("pirate-take-over") do |task|
+      blog.posts.each do |post|                                     
+        task.add_job ConvertPostToPirateTalk, post.id
+      end
+      
+      task.add_finalization_job CommitSolr
+      task.add_finalization_job UnlockBlog, blog.id
+    end    
+
+This would convert all the posts to pirate talk in parallel, using as
+many workers as are available.  Once all the normal jobs are completed
+the finalization jobs are run serially in a single worker.
+Finalization are executed in the order in which they are registered.
+In our example, solr will be committed and then, after the commit is
+complete, the blog will be unlocked.
+
+Details
+----
+
+MultiStepTask creates a queue in resque for each task.  To process
+multi-step jobs you will need at least one Resque worker with
+`QUEUES=*`.  This combined with [resque-fairly][] provides fair
+scheduling of the constituent jobs.  
+
+Having a queue per multi-step task means that is easy to determine to
+what task a particular job belongs. It also provides a nice way to see
+what is going on in the system at any given time.  Just got to
+resque-web and look the queue list.  Use meaningful slugs for your
+tasks and you get a quick birds-eye view of what is going on.
+
+Note on Patches/Pull Requests
+----
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Update history to reflect the change.
+* Commit, do not mess with rakefile, version.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+Copyright
+-----
+
+Copyright (c) OpenLogic, Peter Williams. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,72 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "resque-multi-step"
+    gem.summary = "Provides multi-step tasks with finalization and progress tracking"
+    gem.description = "Provides multi-step tasks with finalization and progress tracking"
+    gem.email = "pezra@barelyenough.org"
+    gem.homepage = "http://github.com/pezra/resque-multi-step"
+    gem.authors = ["Peter Williams", "Morgan Whitney"]
+
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+
+    gem.add_dependency 'redis-namespace', '~> 0.8.0'
+    gem.add_dependency 'resque', '~> 1.10'
+    gem.add_dependency 'resque-fairly', '~> 1.0'
+
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+namespace(:spec) do 
+  Spec::Rake::SpecTask.new(:acceptance) do |spec|
+    spec.libs << 'lib' << 'spec'
+    spec.spec_files = FileList['spec/acceptance/*_spec.rb']
+  end
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "resque-multi-step #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+# Setup  for acceptance testing
+require 'rubygems'
+require 'resque/tasks'
+require 'resque-fairly'
+
+Resque.redis.namespace = ENV['NAMESPACE'] if ENV['NAMESPACE']
+
+$LOAD_PATH << File.expand_path("lib", File.dirname(__FILE__))
+require 'resque-multi-step'
+
+$LOAD_PATH << File.expand_path("spec/acceptance", File.dirname(__FILE__))
+require 'acceptance_jobs'
+

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/lib/resque/plugins/multi_step_task/assure_finalization.rb

@@ -0,0 +1,15 @@
+module Resque
+  module Plugins
+    class MultiStepTask
+      # in the case that all normal jobs have completed before the job group
+      # is finalized, the job group will never receive the hook to enter
+      # finalizataion.  To avoid this, an AssureFinalization job will be added
+      # to the queue for the sole purposed of initiating finalization for certain.
+      class AssureFinalization
+        def self.perform(task_id)
+          MultiStepTask.find(task_id).maybe_finalize
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/multi_step_task/atomic_counters.rb

@@ -0,0 +1,21 @@
+module Resque
+  module Plugins
+    class MultiStepTask
+      module AtomicCounters
+        def counter(name)
+          class_eval <<-INCR
+            def increment_#{name}
+              redis.incrby('#{name}', 1)
+            end
+            INCR
+
+          class_eval <<-GETTER
+            def #{name}
+              redis.get('#{name}').to_i
+            end
+            GETTER
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/multi_step_task/constantization.rb

@@ -0,0 +1,20 @@
+module Resque
+  module Plugins
+    class MultiStepTask
+      module Constantization
+        # Courtesy ActiveSupport (Ruby on Rails) 
+        def constantize(camel_cased_word)
+          names = camel_cased_word.split('::')
+          names.shift if names.empty? || names.first.empty?
+          
+          constant = Object
+          names.each do |name|
+            constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+          end
+          constant
+        end
+      end
+    end
+  end
+end
+

data/lib/resque/plugins/multi_step_task/finalization_job.rb

@@ -0,0 +1,38 @@
+require 'resque/plugins/multi_step_task/constantization'
+
+module Resque
+  module Plugins
+    class MultiStepTask
+      # Executes a single finalization job 
+      class FinalizationJob
+        extend Constantization
+
+        # Handle job invocation
+        def self.perform(task_id, job_module_name, *args)
+#          puts "finalizationjob#perform(#{task_id.inspect}, #{job_module_name.inspect}, #{args.inspect})"
+          task = MultiStepTask.find(task_id)
+          
+          begin
+            constantize(job_module_name).perform(*args)
+          rescue Exception
+            task.increment_failed_count
+            raise
+          end
+          
+          task.increment_completed_count
+
+          if fin_job_info = task.redis.lpop('finalize_jobs')
+            # Queue the next finalization job
+            Resque::Job.create(task.queue_name, FinalizationJob, task.task_id, 
+                               *Yajl::Parser.parse(fin_job_info))
+          else
+            # There is nothing left to do so cleanup.
+            task.nuke
+          end
+
+        end
+      end
+    end
+  end
+end
+

data/lib/resque/plugins/multi_step_task.rb

@@ -0,0 +1,268 @@
+require 'resque'
+require 'redis-namespace'
+require 'resque/plugins/multi_step_task/assure_finalization'
+require 'resque/plugins/multi_step_task/finalization_job'
+require 'resque/plugins/multi_step_task/constantization'
+require 'resque/plugins/multi_step_task/atomic_counters'
+
+module Resque
+  module Plugins
+    class MultiStepTask
+      class NoSuchMultiStepTask < StandardError; end
+      class NotReadyForFinalization < StandardError; end
+      class FinalizationAlreadyBegun < StandardError; end
+
+      class << self
+        include Constantization
+        
+        NONCE_CHARS = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a
+
+        # A bit of randomness to ensure tasks are uniquely identified.
+        def nonce
+          nonce = ""
+          5.times{nonce << NONCE_CHARS[rand(NONCE_CHARS.length)]}
+          nonce
+        end
+        
+        # A redis client suitable for storing global mutli-step task info.
+        def redis
+          @redis ||= Redis::Namespace.new("resque:multisteptask", :redis => Resque.redis)
+        end
+
+        # Does a task with the specified id exist?
+        def active?(task_id)
+          redis.sismember("active-tasks", task_id)
+        end
+        
+        # Create a brand new parallel job group.
+        #
+        # @param [#to_s] slug The descriptive slug of the new job.  Default: a
+        #   random UUID
+        #
+        # @yield [multi_step_task] A block to define the work to take place in parallel
+        #
+        # @yieldparam [MultiStepTask] The newly create job group.
+        #
+        # @return [MultiStepTask] The new job group
+        def create(slug=nil)
+          task_id = if slug.nil? || slug.empty?
+                      "multi-step-task" 
+                    else
+                      slug.to_s
+                    end
+          task_id << "~" << nonce
+          
+          pjg = new(task_id)
+          pjg.nuke
+          redis.sadd("active-tasks", task_id)
+          redis.sismember("active-tasks", task_id)
+          if block_given?
+            yield pjg
+            pjg.finalizable!
+          end
+          
+          pjg
+        end
+
+        # Prevent calling MultiStepTask.new
+        private :new
+        
+        # Find an existing MultiStepTask.
+        #
+        # @param [#to_s] task_id The unique key for the job group of interest.
+        #
+        # @return [ParallelJobGroup] The group of interest
+        #
+        # @raise [NoSuchMultiStepTask] If there is not a group with the specified key.
+        def find(task_id)
+          raise NoSuchMultiStepTask unless active?(task_id)
+          
+          pjg = new(task_id)
+        end
+
+        # Handle job invocation
+        def perform(task_id, job_module_name, *args)
+          task = MultiStepTask.find(task_id)
+          begin
+            constantize(job_module_name).perform(*args)
+          rescue Exception => e
+            task.increment_failed_count
+            raise
+          end
+
+          task.increment_completed_count
+          task.maybe_finalize
+        end
+
+        # Normally jobs that are part of a multi-step task are run
+        # asynchronously by putting them on a queue.  However, it is
+        # often more convenient to just run the jobs synchronously as
+        # they are registered in a development environment.  Setting
+        # mode to `:sync` provides a way to do just that.
+        #
+        # @param [:sync,:async] sync_or_async
+        def mode=(sync_or_async)
+          @@synchronous = (sync_or_async == :sync)
+        end
+
+        def synchronous?
+          @@synchronous
+        end
+        @@synchronous = false
+      end
+
+      def synchronous?
+        @@synchronous
+      end
+
+      # Instance methods 
+
+      include Constantization
+
+      attr_reader :task_id
+
+      extend AtomicCounters
+
+      counter :normal_job_count
+      counter :finalize_job_count
+
+      counter :completed_count
+      counter :failed_count
+
+
+      # Initialize a newly instantiated parallel job group.
+      #
+      # @param [String] task_id The UUID of the group of interest.
+      def initialize(task_id)
+        @task_id = task_id
+      end
+
+      def redis
+        @redis ||= Redis::Namespace.new("resque:multisteptask:#{task_id}", :redis => Resque.redis)
+      end
+
+      # The total number of jobs that are part of this task.
+      def total_job_count
+        normal_job_count + finalize_job_count
+      end
+
+      # Removes all data from redis related to this task.
+      def nuke
+        redis.keys('*').each{|k| redis.del k}
+        Resque.remove_queue queue_name
+        self.class.redis.srem('active-tasks', task_id)
+      end
+      
+      # The name of the queue for jobs what are part of this task.
+      def queue_name
+        task_id
+      end
+
+      # Add a job to this task
+      #
+      # @param [Class,Module] job_type The type of the job to be performed.
+      def add_job(job_type, *args)
+        increment_normal_job_count
+
+        if synchronous?
+          self.class.perform(task_id, job_type.to_s, *args)
+        else
+          Resque::Job.create(queue_name, self.class, task_id, job_type.to_s, *args)
+        end
+      end
+
+      # Finalization jobs are performed after all the normal jobs
+      # (i.e. the ones registered with #add_job) have been completed.
+      # Finalization jobs are performed in the order they are defined.
+      #
+      # @param [Class,Module] job_type The type of job to be performed.
+      def add_finalization_job(job_type, *args)
+        increment_finalize_job_count
+
+        redis.rpush 'finalize_jobs', Yajl::Encoder.encode([job_type.to_s, *args])
+      end
+
+      # A multi-step task is finalizable when all the normal jobs (see
+      # #add_job) have been registered.  Finalization jobs will not be
+      # executed until the task becomes finalizable regardless of the
+      # number of jobs that have been completed.
+      def finalizable?
+        redis.exists 'is_finalizable'
+      end
+
+      # Make this multi-step task finalizable (see #finalizable?).
+      def finalizable!
+        redis.set 'is_finalizable', true
+        if synchronous?
+          maybe_finalize
+        else
+          Resque::Job.create(queue_name, AssureFinalization, self.task_id)
+        end
+      end
+
+      # Finalize this job group.  Finalization entails running all
+      # finalization jobs serially in the order they were defined.
+      #
+      # @raise [NotReadyForFinalization] When called before all normal
+      #   jobs have been attempted.
+      #
+      # @raise [FinalizationAlreadyBegun] If some other process has
+      #   already started (and/or finished) the finalization process.
+      def finalize!
+        raise FinalizationAlreadyBegun unless MultiStepTask.active?(task_id)
+        raise NotReadyForFinalization if !ready_for_finalization? || incomplete_because_of_errors?
+
+        # Only one process is allowed to start the finalization
+        # process.  This setnx acts a global mutex for other processes
+        # that finish about the same time.
+        raise FinalizationAlreadyBegun unless redis.setnx("i_am_the_finalizer", 1)
+        
+        if synchronous?
+          sync_finalize!
+
+        else
+          if fin_job_info = redis.lpop('finalize_jobs')
+            fin_job_info = Yajl::Parser.parse(fin_job_info)
+            Resque::Job.create(queue_name, FinalizationJob, self.task_id, *fin_job_info)
+          else
+            # There is nothing left to do so cleanup.
+            nuke
+          end
+        end
+      end
+
+      def sync_finalize!
+        while fin_job_info = redis.lpop('finalize_jobs')
+          job_class_name, *args = Yajl::Parser.parse(fin_job_info)
+          self.class.perform(task_id, job_class_name, *args)
+        end
+      end
+
+      # Execute finalization sequence if it is time.
+      def maybe_finalize
+        return unless ready_for_finalization?
+        finalize!
+      rescue Exception
+        # just eat the exception
+      end
+
+      # Is this task at the point where finalization can occur.
+      def ready_for_finalization?
+        finalizable? && completed_count >= normal_job_count
+      end
+
+      # If a normal or finalization job fails (i.e. raises an
+      # exception) the task as a whole is considered to be incomplete.
+      # The finalization sequence will not be performed.  If the
+      # failure occurred during finalization any remaining
+      # finalization job will not be run.
+      #
+      # If the failed job is retried and succeeds finalization will
+      # proceed at usual.
+      def incomplete_because_of_errors?
+        failed_count > 0 && completed_count < normal_job_count
+      end
+    end
+  end
+end
+

data/lib/resque-multi-step.rb

@@ -0,0 +1,2 @@
+require 'rubygems'
+require 'resque/plugins/multi_step_task'

data/lib/resque_mutli_step.rb

@@ -0,0 +1 @@
+require 'resque-multi-step'

data/resque-multi-step-task.gemspec

@@ -0,0 +1,67 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{resque-multi-step-task}
+  s.version = "0.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Peter Williams"]
+  s.date = %q{2010-09-20}
+  s.description = %q{Provides multi-step tasks with finalization and progress tracking}
+  s.email = %q{pezra@barelyenough.org}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.md"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.md",
+     "Rakefile",
+     "VERSION",
+     "lib/resque-multi-step-task.rb",
+     "lib/resque/plugins/multi_step_task.rb",
+     "lib/resque/plugins/multi_step_task/assure_finalization.rb",
+     "lib/resque/plugins/multi_step_task/atomic_counters.rb",
+     "lib/resque/plugins/multi_step_task/constantization.rb",
+     "lib/resque_mutli_step_task.rb",
+     "spec/resque-multi-step-task_spec.rb",
+     "spec/resque/plugins/multi_step_task_spec.rb",
+     "spec/spec.opts",
+     "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/pezra/resque-multi-step-task}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{Provides multi-step tasks with finalization and progress tracking}
+  s.test_files = [
+    "spec/resque/plugins/multi_step_task_spec.rb",
+     "spec/resque-multi-step-task_spec.rb",
+     "spec/spec_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
+      s.add_runtime_dependency(%q<resque>, ["~> 1.10"])
+      s.add_runtime_dependency(%q<redis-namespace>, ["~> 0.8.0"])
+    else
+      s.add_dependency(%q<rspec>, [">= 1.2.9"])
+      s.add_dependency(%q<resque>, ["~> 1.10"])
+      s.add_dependency(%q<redis-namespace>, ["~> 0.8.0"])
+    end
+  else
+    s.add_dependency(%q<rspec>, [">= 1.2.9"])
+    s.add_dependency(%q<resque>, ["~> 1.10"])
+    s.add_dependency(%q<redis-namespace>, ["~> 0.8.0"])
+  end
+end
+