resque-sliders 0.0.4

data/Gemfile

@@ -0,0 +1,9 @@
+source "http://rubygems.org"
+#source "http://gems.github.com"
+
+gem 'resque', '1.15.0'
+gem 'resque-sliders'
+
+group :development do
+  gem "shotgun"
+end

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) Kevin Mullin
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,88 @@
+Resque Sliders
+==============
+
+[github.com/kmullin/resque-sliders](https://github.com/kmullin/resque-sliders)
+
+Description
+-----------
+
+ResqueSliders is a [Resque](https://github.com/defunkt/resque) plugin which allows you
+to control Resque workers from the Web-UI.
+
+From the Resque-Web UI, you can:
+
+* Start workers with any queue, or combination of queues on any host, and specify how many of each should be running
+* Pause / Stop / Restart ALL running workers
+
+
+ResqueSliders comes with two parts:
+
+* `KEWatcher`: A daemon that runs on any machine that needs to run Resque workers, watches over the workers and controls which ones are running
+* `Resque-Web Plugin`: A bunch of slider bars, with text-input box to specify what queues to run on the workers
+
+Installation
+------------
+
+Install as a gem:
+
+```
+$ gem install resque-sliders
+```
+
+KEWatcher
+---------
+This is the daemon component that runs on any host that you want to run Resque workers on. The daemon's job is to manage how many Resque workers should be running, and what they should be running. It also provides an easy way to stop all workers during maintenance or deploys.
+
+When the daemon first runs, it will register itself, by hostname with Redis:
+
+* Adds a few persistent settings to the hash key `resque:plugins:resque-sliders:host_configs` (max_children, current_children)
+* Gets any queues that need to be running on the host by looking at `resque:plugins:resque-sliders:` + `hostname`
+
+```
+Usage: kewatcher [options]
+
+Options:
+    -c, --config CONFIG              Resque Config (Yaml)
+    -r, --rakefile RAKEFILE          Rakefile location
+    -p, --pidfile PIDFILE            PID File location
+    -f, --force                      FORCE KILL ANY OTHER RUNNING KEWATCHERS
+    -v, --verbose                    Verbosity (Can be specified more than once, -vv)
+    -m, --max MAX                    Max Children
+    -h, --help                       This help
+```
+
+**Important Options**
+
+* `Max Children (-m|--max MAX)`: Maximum number of workers to run on host (default: 10)
+* `Rakefile (-r|--rakefile RAKEFILE)`: Pass along a rakefile to use when calling `rake ... resque:work` - shouldn't be needed if run from project directory
+* `Force (-f|--force)`: Force any currently running KEWatcher processes to QUIT, waiting for it to do so, and starting in its place
+
+**Controlling the Daemon**
+
+`KEWatcher` supports all the same signals as `Resque`:
+
+* `TERM`, `INT`, `QUIT`: Shutdown. Gracefully kill all child Resque workers, and wait for them to finish before exiting
+* `HUP`: Restart all Resque workers by gracefully killing them, and starting new ones in their place
+* `USR1`: Stop all Resque workers, and don't start any more
+* `USR2`: Pause spawning of new queues, but leave current ones running
+* `CONT`: Unpause. Continue spawning/managing child Resque workers
+
+
+Resque-Web Integration
+----------------------
+**Main Screen:** showing 3 hosts (node01-03), and showing that nodes 1 and 3 aren't running their KEWatchers
+![Screen 1](https://github.com/kmullin/resque-sliders/raw/master/misc/resque-sliders_main-view.png)
+
+**Host Screen:** showing 3 different `QUEUE` combinations (comma separated) and slider bars indicating how many of each of them should run on node02
+![Screen 2](https://github.com/kmullin/resque-sliders/raw/master/misc/resque-sliders_host-view.png)
+
+To enable the Resque-Web Integration you'll need to load ResqueSliders to enable the Sliders tab. Just add:
+
+```ruby
+require 'resque-sliders'
+```
+to a file, like resque-web_init.rb, and run resque-web:
+
+```
+resque-web resque-web_init.rb
+```

data/Rakefile

@@ -0,0 +1,35 @@
+require 'bundler/gem_tasks'
+require 'resque/tasks'
+#
+# Setup
+#
+
+$LOAD_PATH.unshift 'lib'
+
+def command?(command)
+  system("type #{command} > /dev/null 2>&1")
+end
+
+
+#
+# Tests
+#
+
+task :default => :test
+
+desc "Run the test suite"
+task :test do
+  rg = command?(:rg)
+  Dir['test/**/*_test.rb'].each do |f|
+    rg ? sh("rg #{f}") : ruby(f)
+  end
+end
+
+desc "Bump version"
+task :git_tag_version do
+  require 'resque-sliders/version'
+  git_tags = `git tag -l`.split.map { |x| x.gsub(/v/, '') }
+  version = Resque::Plugins::ResqueSliders::Version
+  commit_sha = `git log -1 HEAD|head -n1|awk '{print $2}'`
+  (puts version, commit_sha; `git tag v#{version} #{commit_sha}`) unless git_tags.include?(version)
+end

data/bin/kewatcher

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+
+require 'rubygems'
+require 'yaml'
+require 'optparse'
+require 'resque-sliders/kewatcher'
+
+options = {
+  :verbosity => 0,
+  :max_children => 10
+}
+
+OptionParser.new do |opt|
+  opt.banner = "Usage: #{File.basename($0)} [options]"
+
+  opt.separator ""
+  opt.separator "Options:"
+
+  opt.on("-c", "--config CONFIG", "Resque Config (Yaml)") do |config|
+    options[:config] = YAML.load_file(File.expand_path(config)) rescue nil
+    options[:config] ||= config
+  end
+
+  opt.on("-r", "--rakefile RAKEFILE", "Rakefile location") do |rakefile|
+    options[:rakefile] = rakefile
+  end
+
+  opt.on("-p", "--pidfile PIDFILE", "PID File location") do |pidfile|
+    options[:pidfile] = pidfile
+  end
+
+  opt.on("-f", "--force", "FORCE KILL ANY OTHER RUNNING KEWATCHERS") do
+    options[:force] = true
+  end
+
+  opt.on("-v", "--verbose", "Verbosity") do
+    options[:verbosity] += 1
+  end
+
+  opt.on("-m", "--max MAX", "Max Children") do |max|
+    options[:max_children] = max.to_i
+  end
+
+  opt.on("-h", "--help", "This help") do
+    puts opt
+    exit
+  end
+
+end.parse!
+
+begin
+  options[:config] = case options[:config]
+  when Hash
+    options[:config][ENV['RAILS_ENV'] || 'development']
+  when String
+    options[:config]
+  end
+rescue Object => e
+  puts e
+  exit 1
+end
+
+options[:config] ||= 'localhost:6379'
+
+kewatcher = Resque::Plugins::ResqueSliders::KEWatcher.new(options)
+kewatcher.run!

data/config/config_example.yml

@@ -0,0 +1,11 @@
+development:
+  localhost:6379
+
+test:
+  localhost:6379
+
+staging:
+  localhost:6379
+
+production:
+  192.168.0.1:6379

data/lib/resque-sliders/commander.rb

@@ -0,0 +1,73 @@
+module Resque
+  module Plugins
+    module ResqueSliders
+      class Commander
+
+        include Helpers
+
+        # Hosts that have config data (queues and values), but the host is not running the daemon.
+        attr_reader :stale_hosts
+
+        def initialize
+          @host_status = redis_get_hash(host_config_key)
+          @stale_hosts = Resque.redis.keys("#{key_prefix}:*").map { |x| y = x.split(':').last; y unless x == host_config_key or hosts.include?(y) }.compact.sort
+        end
+
+        # Return Array of currently online hosts
+        def hosts
+          @host_status.keys.select { |x| x unless x.split(':').last == 'reload' }.map { |x| x.split(':').first }.uniq.sort
+        end
+
+        # Array of all hosts (current + stale)
+        def all_hosts
+          (hosts + stale_hosts).sort
+        end
+
+        # Return current children count or nil if Host hasn't registered itself.
+        def current_children(host)
+          @host_status["#{host}:current_children"].to_i if max_children(host)
+        end
+
+        # Return max children count or nil if Host hasn't registered itself.
+        def max_children(host)
+          max = @host_status["#{host}:max_children"].to_i
+          max == 0 ? nil : max # if Max isn't set its not running
+        end
+
+        # Override max_children on host (Dangerous!)
+        def max_children!(host, count)
+          @hostname = host
+          register_setting('max_children', count)
+        end
+
+        # Return Array of queues on host
+        def queues_on(host)
+          queue_values(host).keys if all_hosts.include?(host)
+        end
+
+        # Changes queues to quantiy for host.
+        # Returns boolean.
+        def change(host, queue, quantity)
+          # queue is sanitized by:
+          # replacing punctuation with spaces, strip end spaces, split on remaining whitespace, and join again on comma.
+          # FIXME
+          queue2 = queue.downcase.gsub(/[^a-z 0-9,_\-\*]/, '').strip.split(/, */).reject {|x| (x.length == 1 and %w(- _).include?(x)) or x.empty? }.join(',')
+          raise 'Queue Different' unless queue == queue2
+          redis_set_hash("#{key_prefix}:#{host}", queue2, quantity) unless queue2.empty?
+        end
+
+        # Deletes queue for host.
+        # Returns boolean.
+        def delete(host, queue)
+          redis_del_hash("#{key_prefix}:#{host}", queue)
+        end
+
+        # Sets reload flag on field of config_key to reload host's KEWatcher
+        def reload(host)
+          redis_set_hash(host_config_key, "#{host}:reload", 1)
+        end
+
+      end
+    end
+  end
+end

data/lib/resque-sliders/helpers.rb

@@ -0,0 +1,72 @@
+module Resque
+  module Plugins
+    module ResqueSliders
+      module Helpers
+
+        def key_prefix
+          "plugins:resque-sliders"
+        end
+
+        def host_config_key
+          "plugins:resque-sliders:host_configs"
+        end
+
+        def redis_get_hash(key)
+          Resque.redis.hgetall(key)
+        end
+
+        def redis_get_hash_field(key, field)
+          Resque.redis.hget(key, field)
+        end
+
+        def redis_set_hash(key, field, fvalue)
+          Resque.redis.hset(key, field, fvalue) == 1 ? true : false
+        end
+
+        def redis_del_hash(key, field)
+          Resque.redis.hdel(key, field) == 1 ? true : false
+        end
+
+        # Return Hash: { queue => # }
+        def queue_values(host)
+          redis_get_hash("#{key_prefix}:#{host}")
+        end
+
+        def register_setting(setting, value)
+          redis_set_hash(host_config_key, "#{@hostname}:#{setting}", value)
+        end
+
+
+        # Signal Checking
+
+
+        # Gets signal field in redis config_key for this host. Don't call directly
+        def check_signal(host)
+          sig = caller[0][/`([^']*)'/, 1].gsub('?', '')
+          raise 'Dont call me that' unless %w(reload pause stop).include?(sig)
+          redis_get_hash_field(host_config_key, "#{host}:#{sig}").to_i == 1 ? true : false
+        end
+
+        def reload?(host)
+          check_signal(host)
+        end
+
+        def pause?(host)
+          check_signal(host)
+        end
+
+        def stop?(host)
+          check_signal(host)
+        end
+
+        # Return Hash of signals for host and their values
+        #def check_redis_for_signals(host)
+        #  configs = redis_get_hash(host_config_key)
+        #  signals = %w(reload pause stop).map { |x| [host,x].join(':') }
+        #  Hash[configs.delete_if { |k,v| ! signals.include?(k) }.map { |k,v| [k.split(':').last.to_sym ,v] }]
+        #end
+
+      end
+    end
+  end
+end