resque-director 1.0.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,11 @@
+source "http://rubygems.org"
+
+gem "json"
+gem 'resque'
+
+group :development do
+  gem "rspec", "~> 2.3.0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.6.4"
+  gem "rcov", ">= 0"
+end

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Nolan Frausto
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,76 @@
+= Resque Director
+
+Resque Director is a plugin for the Resque queueing system (http://github.com/defunkt/resque) that "directs" workers on a queue by automatically adding or removing workers from a queue based on how backed up a queue becomes, or how long it takes for a job to get pulled off the queue.
+
+==About
+
+resque-director is mainly useful for when you are managing a large number of workers and don't want to waste resources keeping all of them waiting when they are not being used. Also useful in queues where the influx of jobs can change dramatically from time to time: enabling more workers during the times when the queue is filling up more quickly, and less in the opposite scenario. Different queues can be given different directions as well.
+
+== Usage
+
+When creating your jobs you should include (make sure to include not extend) Resque::Plugins::Director and add direction options. 
+
+For Example:
+	
+  class Job
+    include Resque::Plugins::Director
+    direct :min_workers => 2, :max_workers => 4, :max_time => 60, :max_queue => 10, :wait_time => 30
+    @queue = :test
+    
+    #rest of your Job class here
+  end
+  
+
+=== Configuration Options
+
+<b>min_workers</b>:: specifies the minimum number of workers running at any point in time. If there are no workers running or less than the minimum running it will start as many workers necessary to get it to the minimum. The default is 1.
+
+<b>max_workers</b>:: specifies the maximum number of workers running at any point in time. It will never start more than the maximum number of workers. If anything less than or equal to zero is specified as the maximum it will be treated as if there is no maximum, and theoretically an infinite number of workers could be added. The default is 0.
+
+<b>max_time</b>:: the maximum time in seconds that a job takes to get pulled off the queue, if a job takes longer than this time then a worker is added. If anything less than or equal to zero is specified as the maximum time, this field will be ignored. The default is 0.
+
+<b>max_queue</b>:: the maximum jobs that can build up in a queue, if more than this number of jobs build up then a worker is added. If anything less than or equal to zero is specified as the maximum queue, this field will be ignored. The default is 0.
+
+<b>wait_time</b>:: the time that it will wait after adding or removing a worker before being allowed to add or remove workers again. The default is 60 seconds.
+
+== Worker Options
+
+<b>start_override</b>:: This will run exactly what you put in the command override as a system command to start A SINGLE worker, allowing you to customize the starting of a worker fully. The system command "QUEUE=queue_name rake resque:work" is run by default, where queue_name is whatever queue the job is running.
+
+<b>stop_override</b>:: This will run exactly what you put in the command override as a system command to stop A SINGLE worker, allowing you to customize the stoping of a worker fully. Process.kill("QUIT", worker_pid) is used to stop the worker by default, where worker_pid is the PID of a worker.
+
+===	Starting/Stopping Workers Example
+
+  class Job
+    include Resque::Plugins::Director
+    direct :start_override => "./start_command_to_run", :stop_override => "./stop_command_to_run"
+    @queue = :test
+    
+    #rest of your Job class here
+  end
+
+=== Conditions For Removing Workers
+
+A worker will be removed if the jobs in the queue fall below half of the <b>max_queue</b>, or if the time it takes for a job to be pulled off of a queue falls below half of the <b>max_time</b>. Workers will be scaled down to the minimum if there are no jobs on the queue.
+
+=== Special Cases
+
+* If a max_worker is less than min_worker then the default for max_worker will be used (there will be no maximum). 
+* If a min_workers is set to anything less than 1 then it will be treated as 0.
+
+
+== Contributing to resque-reconnect
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2011 Nolan Frausto. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,52 @@
+# encoding: utf-8
+$LOAD_PATH.unshift 'lib'
+
+require 'rubygems'
+require 'bundler'
+require 'resque/tasks'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "resque-director"
+  gem.homepage = "http://github.com/frausto/resque-director"
+  gem.license = "MIT"
+  gem.summary = %Q{resque plugin for dynamically adding/removing workers to a queue}
+  gem.description = %Q{resque plugin for dynamically adding/removing workers to a queue}
+  gem.email = "nrfrausto@gmail.com"
+  gem.authors = ["Nolan Frausto"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "resque-director #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/lib/resque/plugins/director/config.rb

@@ -0,0 +1,39 @@
+module Resque
+  module Plugins
+    module Director
+      module Config
+        extend self
+        
+        attr_accessor :queue
+        
+        DEFAULT_OPTIONS = {
+          :min_workers    => 1,
+          :max_workers    => 0,
+          :max_time       => 0,
+          :max_queue      => 0,
+          :wait_time      => 60,
+          :start_override => nil,
+          :stop_override  => nil
+        }
+        
+        def reset!
+          DEFAULT_OPTIONS.each do |key, default|
+            attr_reader key
+            self.instance_variable_set("@#{key.to_s}", default)
+          end
+        end
+
+        def setup(options={})
+          DEFAULT_OPTIONS.each do |key, value|
+            self.instance_variable_set("@#{key.to_s}", options[key] || value)
+          end
+          
+          @min_workers = 0 if @min_workers < 0
+          @max_workers = DEFAULT_OPTIONS[:max_workers] if @max_workers < @min_workers
+        end
+        
+        reset!
+      end
+    end
+  end
+end

data/lib/resque/plugins/director/lifecycle.rb

@@ -0,0 +1,30 @@
+module Resque
+  module Plugins
+    module Director
+      module Lifecycle
+
+        def self.included(base) #:nodoc:
+          base.class_eval do
+            alias_method :push_without_lifecycle, :push
+            extend ClassMethods
+          end
+        end
+
+        module ClassMethods
+
+          def push(queue, item)
+            if item.respond_to?(:[]=)
+              timestamp = {'resdirecttime' => Time.now.utc.to_i}
+              item[:args] = item[:args].push(timestamp)
+            end
+            push_without_lifecycle queue, item
+          end
+        end
+      end
+    end
+  end
+end
+
+module Resque
+  include Resque::Plugins::Director::Lifecycle
+end

data/lib/resque/plugins/director/scaler.rb

@@ -0,0 +1,72 @@
+module Resque
+  module Plugins
+    module Director
+      class Scaler
+        class << self
+        
+          def scale_up(number_of_workers=1)
+            number_of_workers = WorkerTracker.new.total_to_add(number_of_workers)
+            scaling(number_of_workers) do
+              number_of_workers.times { start }
+            end
+          end
+        
+          def scale_down(number_of_workers=1)
+            tracker = WorkerTracker.new
+            number_of_workers = tracker.total_to_remove(number_of_workers)
+            
+            scaling(number_of_workers) do
+              stop(tracker, number_of_workers)
+            end
+          end
+          
+          def scale_down_to_minimum
+            tracker = WorkerTracker.new
+            number_of_workers = tracker.total_to_go_to_minimum
+            stop(tracker, number_of_workers)
+          end
+          
+          def scale_within_requirements
+            number_of_workers = WorkerTracker.new.total_for_requirements
+            
+            if number_of_workers > 0
+              scale_up(number_of_workers)
+            elsif number_of_workers < 0
+              scale_down(number_of_workers * -1)
+            end
+          end
+          
+          def scaling(number_of_workers=1)
+            return unless time_to_scale? && number_of_workers > 0
+            yield
+            Resque.redis.set("last_scaled_#{Config.queue}", Time.now.utc.to_i)
+          end
+        
+          private
+         
+          def time_to_scale?
+            last_time = Resque.redis.get("last_scaled_#{Config.queue}")
+            return true if last_time.nil?
+            time_passed = (Time.now.utc - Time.at(last_time.to_i).utc)
+            time_passed >= Config.wait_time
+          end
+          
+          def start
+            default_command = "QUEUE=#{Config.queue} rake resque:work &"
+            system(Config.start_override || default_command)
+          end
+          
+          def stop(tracker, number_of_workers)
+            if Config.stop_override
+              number_of_workers.times { system(Config.stop_override) }
+            else
+              valid_workers = tracker.workers.select{|w| w.hostname == `hostname`.chomp}
+              worker_pids = valid_workers[0...number_of_workers].map(&:pid)
+              worker_pids.each {|pid| Process.kill("QUIT", pid)}
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/director/worker_tracker.rb

@@ -0,0 +1,57 @@
+module Resque
+  module Plugins
+    module Director
+      class WorkerTracker
+        attr_reader :workers
+        
+        def initialize
+          @workers = current_workers
+          @number_working = @workers.size
+        end
+          
+        def total_for_requirements
+          start_number = workers_to_start
+          stop_number = workers_to_stop
+          return start_number if start_number > 0 
+          return stop_number if stop_number < 0 
+          0
+        end
+        
+        def total_to_go_to_minimum
+          to_minimum =  @number_working - Config.min_workers
+          to_minimum > 0 ? to_minimum : 0
+        end
+      
+        def total_to_add(number_to_start)
+          return number_to_start if Config.max_workers <= 0
+          scale_limit = Config.max_workers - @number_working
+          number_to_start > scale_limit ? scale_limit : number_to_start
+        end
+        
+        def total_to_remove(number_to_stop)
+          min_workers = Config.min_workers <= 0 ? 1 : Config.min_workers
+          scale_limit = @number_working - min_workers
+          number_to_stop > scale_limit ? scale_limit : number_to_stop
+        end
+        
+        private
+        
+        def workers_to_start
+          min_workers = Config.min_workers <= 0 ? 1 : Config.min_workers
+          workers_to_start = min_workers - @number_working
+        end
+        
+        def workers_to_stop
+          return 0 if Config.max_workers <= 0
+          workers_to_stop = Config.max_workers - @number_working
+        end
+        
+        def current_workers
+          Resque.workers.select do |w|
+            w.queues == [Config.queue] && !w.shutdown?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/director.rb

@@ -0,0 +1,95 @@
+module Resque
+  module Plugins
+    module Director
+      
+      def self.included(base)
+        base.extend ClassMethods
+        base.overwrite_perform
+        base.instance_eval do
+          def singleton_method_added(name)
+            return if name != :perform
+            overwrite_perform
+          end
+        end
+      end
+      
+      module ClassMethods
+        def direct(options={})
+          Config.setup(options)
+        end
+        
+        def overwrite_perform
+          class_eval do |klass|
+            if klass.respond_to?('perform') && !klass.respond_to?('custom_perform')
+              klass.instance_eval do
+                def custom_perform(*args)
+                  args.pop unless retrieve_timestamp(args.last).nil?
+                  original_perform(*args)
+                end
+              end
+
+              class << klass
+                alias_method :original_perform, :perform
+                alias_method :perform, :custom_perform
+              end
+            end
+          end
+        end
+      
+        def after_enqueue_scale_workers(*args)
+          Config.queue = @queue.to_s
+          Scaler.scale_within_requirements
+        end
+      
+        def before_perform_direct_workers(*args)
+          return unless scaling_config_set?
+          Config.queue = @queue.to_s
+          time_stamp = retrieve_timestamp(args.pop)
+          start_time = time_stamp.nil? ? Time.now.utc : Time.at(time_stamp.to_i).utc 
+        
+          time_through_queue = Time.now.utc - start_time
+          jobs_in_queue = Resque.size(@queue.to_s)
+        
+          if scale_up?(time_through_queue, jobs_in_queue)
+            Scaler.scale_up
+          elsif scale_down?(time_through_queue, jobs_in_queue)
+            Scaler.scale_down
+          end
+        end
+      
+        def after_perform_direct_workers(*args)
+          jobs_in_queue = Resque.size(@queue.to_s)
+          Scaler.scale_down_to_minimum if jobs_in_queue == 0
+        end
+      
+        def on_failure_direct_workers(*args)
+          jobs_in_queue = Resque.size(@queue.to_s)
+          Scaler.scale_down_to_minimum if jobs_in_queue == 0
+        end
+      
+        private
+        
+        def retrieve_timestamp(timestamp)
+          return nil unless timestamp.class.to_s == "Hash"
+          timestamp['resdirecttime'] || timestamp[:resdirecttime]
+        end
+      
+        def scaling_config_set?
+          Config.max_time > 0 || Config.max_queue > 0
+        end
+      
+        def scale_up?(time_through_queue, jobs_in_queue)
+          time_limits =  Config.max_time > 0 && time_through_queue > Config.max_time 
+          queue_limits = Config.max_queue > 0 && jobs_in_queue > Config.max_queue
+          time_limits || queue_limits
+        end
+      
+        def scale_down?(time_through_queue, jobs_in_queue)
+          time_limits =  Config.max_time > 0 && time_through_queue < (Config.max_time/2)
+          queue_limits = Config.max_queue > 0 && jobs_in_queue < (Config.max_queue/2) 
+          (Config.max_time <= 0 || time_limits) && (Config.max_queue <= 0 || queue_limits)
+        end
+      end
+    end
+  end
+end

data/lib/resque-director.rb

@@ -0,0 +1,7 @@
+require 'resque'
+
+require 'resque/plugins/director'
+require 'resque/plugins/director/worker_tracker'
+require 'resque/plugins/director/config'
+require 'resque/plugins/director/scaler'
+require 'resque/plugins/director/lifecycle'

data/resque-director.gemspec

@@ -0,0 +1,76 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{resque-director}
+  s.version = "1.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = [%q{Nolan Frausto}]
+  s.date = %q{2011-08-20}
+  s.description = %q{resque plugin for dynamically adding/removing workers to a queue}
+  s.email = %q{nrfrausto@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    "Gemfile",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "lib/resque-director.rb",
+    "lib/resque/plugins/director.rb",
+    "lib/resque/plugins/director/config.rb",
+    "lib/resque/plugins/director/lifecycle.rb",
+    "lib/resque/plugins/director/scaler.rb",
+    "lib/resque/plugins/director/worker_tracker.rb",
+    "resque-director.gemspec",
+    "spec/redis-test.conf",
+    "spec/resque/plugins/director/config_spec.rb",
+    "spec/resque/plugins/director/lifecycle_spec.rb",
+    "spec/resque/plugins/director/scaler_spec.rb",
+    "spec/resque/plugins/director/worker_tracker_spec.rb",
+    "spec/resque/plugins/director_spec.rb",
+    "spec/spec_helper.rb",
+    "spec/support/test_job.rb"
+  ]
+  s.homepage = %q{http://github.com/frausto/resque-director}
+  s.licenses = [%q{MIT}]
+  s.require_paths = [%q{lib}]
+  s.rubygems_version = %q{1.8.8}
+  s.summary = %q{resque plugin for dynamically adding/removing workers to a queue}
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<json>, [">= 0"])
+      s.add_runtime_dependency(%q<resque>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, ["~> 2.3.0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+    else
+      s.add_dependency(%q<json>, [">= 0"])
+      s.add_dependency(%q<resque>, [">= 0"])
+      s.add_dependency(%q<rspec>, ["~> 2.3.0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<json>, [">= 0"])
+    s.add_dependency(%q<resque>, [">= 0"])
+    s.add_dependency(%q<rspec>, ["~> 2.3.0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+  end
+end
+