resque-cluster 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 720fd972bd513c6dc735a733251272c993aca7d4
+  data.tar.gz: 4baeac9c942beb915c36a997ac0845a1d3eeaec9
+SHA512:
+  metadata.gz: 65ffb5acfd27b3b9a561e9be011af40089375136c2aef55707fc11b4c107837589c407347b917ef3c6e570abd62ea965f85191de143045d3b3f06201f3734f4e
+  data.tar.gz: e4a90b5e3b707ae20bb4034ae05272144ad75b1cdcede9d2e67b2b599f67066d56cb3ee27cfdb8f18341534b6c17a58edfbc37f8537d5b81a639f4c694d3f881

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.gitignore

@@ -0,0 +1,49 @@
+# rcov generated
+coverage
+coverage.data
+
+# rdoc generated
+rdoc
+
+# yard generated
+doc
+.yardoc
+
+# bundler
+.bundle
+
+# jeweler generated
+pkg
+
+# Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore: 
+#
+# * Create a file at ~/.gitignore
+# * Include files you want ignored
+# * Run: git config --global core.excludesfile ~/.gitignore
+#
+# After doing this, these files will be ignored in all your git projects,
+# saving you from having to 'pollute' every project you touch with them
+#
+# Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line)
+#
+# For MacOS:
+#
+#.DS_Store
+
+# For TextMate
+#*.tmproj
+#tmtags
+
+# For emacs:
+#*~
+#\#*
+#.\#*
+
+# For vim:
+#*.swp
+
+# For redcar:
+#.redcar
+
+# For rubinius:
+#*.rbc

data/.rspec

@@ -0,0 +1 @@
+--color

data/.rubocop.yml

@@ -0,0 +1,7 @@
+AllCops:
+  Exclude:
+    - bin/**
+    - spec/**
+
+LineLength:
+  Enabled: false

data/Gemfile

@@ -0,0 +1,16 @@
+source 'http://rubygems.org'
+
+gem 'resque-pool', '~> 0.5.0'
+gem 'gru', '0.0.3'
+
+group :development do
+  gem 'pry'
+  gem 'awesome_print'
+  gem 'rspec', '~> 3.1.0'
+  gem 'rdoc', '~> 3.12'
+  gem 'bundler', '~> 1.0'
+  gem 'jeweler', '~> 2.0.1'
+  gem 'simplecov', '>= 0'
+  gem 'rubocop', '~> 0.31'
+  gem 'mock_redis', '~> 0.15.0'
+end

data/Gemfile.lock

@@ -0,0 +1,135 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.3.8)
+    ast (2.0.0)
+    astrolabe (1.3.0)
+      parser (>= 2.2.0.pre.3, < 3.0)
+    awesome_print (1.6.1)
+    builder (3.2.2)
+    coderay (1.1.0)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    faraday (0.9.1)
+      multipart-post (>= 1.2, < 3)
+    git (1.2.9.1)
+    github_api (0.12.3)
+      addressable (~> 2.3)
+      descendants_tracker (~> 0.0.4)
+      faraday (~> 0.8, < 0.10)
+      hashie (>= 3.3)
+      multi_json (>= 1.7.5, < 2.0)
+      nokogiri (~> 1.6.3)
+      oauth2
+    gru (0.0.3)
+      redis (> 0.0)
+    hashie (3.4.1)
+    highline (1.7.2)
+    jeweler (2.0.1)
+      builder
+      bundler (>= 1.0)
+      git (>= 1.2.5)
+      github_api
+      highline (>= 1.6.15)
+      nokogiri (>= 1.5.10)
+      rake
+      rdoc
+    json (1.8.2)
+    jwt (1.5.0)
+    method_source (0.8.2)
+    mini_portile (0.6.2)
+    mock_redis (0.15.0)
+    mono_logger (1.1.0)
+    multi_json (1.11.0)
+    multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    oauth2 (1.0.0)
+      faraday (>= 0.8, < 0.10)
+      jwt (~> 1.0)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    parser (2.2.2.2)
+      ast (>= 1.1, < 3.0)
+    powerpack (0.1.1)
+    pry (0.9.12.6)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rack (1.6.1)
+    rack-protection (1.5.3)
+      rack
+    rainbow (2.0.0)
+    rake (10.4.2)
+    rdoc (3.12.2)
+      json (~> 1.4)
+    redis (3.2.1)
+    redis-namespace (1.5.2)
+      redis (~> 3.0, >= 3.0.4)
+    resque (1.25.2)
+      mono_logger (~> 1.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.3)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    resque-pool (0.5.0)
+      rake
+      resque (~> 1.22)
+      trollop (~> 2.0)
+    rspec (3.1.0)
+      rspec-core (~> 3.1.0)
+      rspec-expectations (~> 3.1.0)
+      rspec-mocks (~> 3.1.0)
+    rspec-core (3.1.7)
+      rspec-support (~> 3.1.0)
+    rspec-expectations (3.1.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.1.0)
+    rspec-mocks (3.1.3)
+      rspec-support (~> 3.1.0)
+    rspec-support (3.1.2)
+    rubocop (0.31.0)
+      astrolabe (~> 1.3)
+      parser (>= 2.2.2.1, < 3.0)
+      powerpack (~> 0.1)
+      rainbow (>= 1.99.1, < 3.0)
+      ruby-progressbar (~> 1.4)
+    ruby-progressbar (1.7.5)
+    simplecov (0.10.0)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    sinatra (1.4.6)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
+    slop (3.5.0)
+    thread_safe (0.3.5)
+    tilt (2.0.1)
+    trollop (2.1.2)
+    vegas (0.1.11)
+      rack (>= 1.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  awesome_print
+  bundler (~> 1.0)
+  gru (= 0.0.3)
+  jeweler (~> 2.0.1)
+  mock_redis (~> 0.15.0)
+  pry
+  rdoc (~> 3.12)
+  resque-pool (~> 0.5.0)
+  rspec (~> 3.1.0)
+  rubocop (~> 0.31)
+  simplecov
+
+BUNDLED WITH
+   1.10.6

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2015 Yasha Portnoy
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,84 @@
+= resque-cluster
+
+*resque-cluster* allows managing resque workers distributed across multiple hosts. It allows user to set the maximum number of workers to be running per host as well as the maximum number of workers running per cluster and maintains correct counts via a shared _Redis_ server.
+
+*resque-cluster* is based on _resque-pool_ gem and will still run in a non clustered mode if, for example, you are anticipating expanding across multiple hosts at a later time.
+
+== Installation
+
+  gem install resque-cluster
+Or, more likely, add it to the Gemfile used by your resque workers
+  gem resque-cluster
+
+== Usage
+
+  bundle exec resque-cluster_member <options>
+
+== Options
+
+  -c, --config=<s>              Path to the config(local/host maximums config in a cluster mode) 
+                                  YAML file. Always required
+  -a, --appname=<s>             Application name
+  -d, --daemon                  Run as a background daemon
+  -o, --stdout=<s>              Redirect stdout to logfile
+  -e, --stderr=<s>              Redirect stderr to logfile
+  -n, --nosync                  Don't sync logfiles on every write
+  -p, --pidfile=<s>             PID file location
+  -E, --environment=<s>         Set RAILS_ENV/RACK_ENV/RESQUE_ENV(in a cluster mode used for
+                                  namespacing the cluster)
+  -s, --spawn-delay=<i>         Delay in milliseconds between spawning missing workers
+  -t, --term-graceful-wait      On TERM signal, wait for workers to shut down gracefully
+  -r, --term-graceful           On TERM signal, shut down workers gracefully
+  -m, --term-immediate          On TERM signal, shut down workers immediately (default)
+  -i, --single-process-group    Workers remain in the same process group as the master
+  -C, --cluster=<s>             Name of the cluster this resque-pool belongs to. Required 
+                                  if you want to launch in a cluster mode
+  -G, --global-config=<s>       Path to the global config file(global/cluster maximums
+                                  config in a cluster mode)
+  -v, --version                 Print version and exit
+
+== Local/Host Config
+
+When run without --cluster option the host config creates precisely the number of workers specified in the config.
+When run with --cluster option set host config specifies the maximum number of workers that can be spun up on the machine
+
+Host config follows the following format:
+  foo: 1
+  bar: 9
+  "foo,bar,baz": 1
+
+== Global/Cluster Config
+
+Cluster config isn't used by a non cluster mode and the param is ignored. In a cluster mode, cluster config defines the maximum number of workers
+that can run across the whole cluster. It also defines the cluster rebalancing mode. When set to true, workers will try to balance the load across all
+cluster members for each worker.
+
+Cluster config follows the following format:
+  global_maximums:
+    foo: 1
+    bar: 9
+    "foo,bar,baz": 1
+  rebalance_cluster: true
+
+== Examples
+
+When trying to launch without a cluster you could use a command like this:
+  bundle exec resque-cluster_member -c config.yml -E production -o /var/log/resque-cluster.stdout.log
+
+When trying to launch with a cluster you could use a command like this:
+  bundle exec resque-cluster_member -c host_maximums.yml -E production -C prod-cluster -G cluster_config.yml -o /var/log/resque-cluster.stdout.log
+
+== Contributing to resque-cluster
+
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
+* Fork the project.
+* Start a feature/bugfix branch.
+* Commit and push until you are happy with your contribution.
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2015 Yasha Portnoy. See LICENSE.txt for
+further details.

data/Rakefile

@@ -0,0 +1,51 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts 'Run `bundle install` to install missing gems'
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification...
+  # see http://guides.rubygems.org/specification-reference/ for more options
+  gem.name = 'resque-cluster'
+  gem.homepage = 'http://github.com/yportnoy/resque-cluster'
+  gem.license = 'MIT'
+  gem.summary = 'This gem sets up a resque pool cluster'
+  gem.description = 'This gem sets up a resque pool cluster'
+  gem.email = 'yportnoy@optoro.com'
+  gem.authors = ['Yasha Portnoy']
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+desc 'Code coverage detail'
+task :simplecov do
+  ENV['COVERAGE'] = 'true'
+  Rake::Task['spec'].execute
+end
+
+task default: :spec
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ''
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "resque-cluster #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/bin/resque-cluster_member

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+# -*- encoding: utf-8 -*-
+
+gem 'resque-pool'
+
+require 'resque/pool/cli'
+
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+
+require 'resque/pool/patches'
+require 'resque/pool/cli_patches'
+require 'resque/cluster'
+
+Resque::Pool::CLI.run

data/lib/resque/cluster.rb

@@ -0,0 +1,14 @@
+require 'resque/cluster/member'
+
+module Resque
+  # Distributed Pool is a clustered resque-pool
+  class Cluster
+    class << self
+      attr_accessor :config, :member
+
+      def init(started_pool)
+        @member = Member.new(started_pool)
+      end
+    end
+  end
+end

data/lib/resque/cluster/member.rb

@@ -0,0 +1,116 @@
+require 'socket'
+require 'gru'
+
+module Resque
+  class Cluster
+    # Member is a single member of a resque pool cluster
+    class Member
+      attr_reader :hostname, :pool, :local_config, :global_config
+
+      def initialize(started_pool)
+        @pool = started_pool
+        @local_config = parse_config(Cluster.config[:local_config_path])
+        @global_config = parse_config(Cluster.config[:global_config_path])
+        @global_config = @local_config if global_config.empty?
+        @worker_count_manager = initialize_gru
+
+        register
+      end
+
+      def perform
+        check_for_worker_count_adjustment
+        ping
+      end
+
+      def register
+        ping
+      end
+
+      def unregister
+        unping
+        remove_counts
+        unqueue_all_workers
+      end
+
+      def check_for_worker_count_adjustment
+        host_count_adjustment = @worker_count_manager.adjust_workers
+        adjust_worker_counts(host_count_adjustment) if host_count_adjustment
+      end
+
+      private
+
+      def global_prefix
+        "cluster:#{Cluster.config[:cluster_name]}:#{Cluster.config[:environment]}"
+      end
+
+      def member_prefix
+        "#{global_prefix}:#{hostname}"
+      end
+
+      def running_workers_key_name
+        "#{member_prefix}:running_workers"
+      end
+
+      def ping
+        Resque.redis.hset(global_prefix, hostname, Time.now.utc)
+      end
+
+      def unping
+        Resque.redis.hdel(global_prefix, hostname)
+      end
+
+      def initialize_gru
+        Gru.create(cluster_member_settings)
+      end
+
+      def hostname
+        @hostname ||= Socket.gethostname
+      end
+
+      def adjust_worker_counts(count_adjustments)
+        count_adjustments.each do |worker, count|
+          @pool.adjust_worker_counts(worker, count)
+          update_counts
+        end
+      end
+
+      def parse_config(config_path)
+        return {} unless config_path && File.exist?(config_path)
+        YAML.load(ERB.new(IO.read(config_path)).result)
+      end
+
+      def remove_counts
+        Resque.redis.del(running_workers_key_name)
+      end
+
+      def unqueue_all_workers
+        @worker_count_manager.release_workers
+      end
+
+      def unqueue_workers(workers)
+        workers = Array(workers)
+        workers.each do |worker|
+          @worker_count_manager.release_workers(worker)
+        end
+      end
+
+      def update_counts
+        current_workers = @pool.config
+        current_workers.each do |key, value|
+          Resque.redis.hset(running_workers_key_name, key, value)
+        end
+      end
+
+      def cluster_member_settings
+        {
+          cluster_maximums: @global_config["global_maximums"] || @global_config,
+          host_maximums:    @local_config,
+          client_settings:  Resque.redis.client.options,
+          rebalance_flag:   @global_config["rebalance_cluster"] || false,
+          cluster_name:     Cluster.config[:cluster_name],
+          environment_name: Cluster.config[:environment]
+        }
+      end
+    end
+  end
+end