resque-cluster 0.0.1

data/lib/resque/cluster/version.rb

@@ -0,0 +1,5 @@
+module Resque
+  class Cluster
+    VERSION = '0.0.1'
+  end
+end

data/lib/resque/pool/cli_patches.rb

@@ -0,0 +1,65 @@
+require 'resque/cluster/member'
+require 'yaml'
+
+module Resque
+  class Pool
+    # Command Line Interface monkey patch for resque-pool
+    module CLI
+      module_function
+
+      original_setup_environment = instance_method(:setup_environment)
+
+      define_method(:setup_environment) do |opts|
+        original_setup_environment.bind(self).call(opts)
+        if opts[:cluster]
+          puts "Starting as a cluster: #{opts[:cluster]} in #{opts[:environment]} environment"
+          Resque::Cluster.config = {
+            cluster_name: opts[:cluster],
+            environment: opts[:environment],
+            local_config_path: opts[:config],
+            global_config_path: opts[:global_config]
+          }
+        end
+      end
+
+      # rubocop:disable all
+      def parse_options
+        opts = Trollop::options do
+          version "resque-pool #{VERSION} (c) nicholas a. evans"
+          banner <<-EOS
+resque-pool is the best way to manage a group (pool) of resque workers
+
+When daemonized, stdout and stderr default to resque-pool.stdxxx.log files in
+the log directory and pidfile defaults to resque-pool.pid in the current dir.
+
+Usage:
+   resque-pool [options]
+where [options] are:
+          EOS
+          opt :config, 'Alternate path to config file', type: String, short: '-c'
+          opt :appname, 'Alternate appname',         type: String,    short: '-a'
+          opt :daemon, 'Run as a background daemon', default: false,  short: '-d'
+          opt :stdout, 'Redirect stdout to logfile', type: String,    short: '-o'
+          opt :stderr, 'Redirect stderr to logfile', type: String,    short: '-e'
+          opt :nosync, 'Don\'t sync logfiles on every write'
+          opt :pidfile, 'PID file location', type: String,    short: '-p'
+          opt :environment, 'Set RAILS_ENV/RACK_ENV/RESQUE_ENV', type: String, short: '-E'
+          opt :spawn_delay, 'Delay in milliseconds between spawning missing workers', type: Integer, short: '-s'
+          opt :term_graceful_wait, 'On TERM signal, wait for workers to shut down gracefully'
+          opt :term_graceful,     'On TERM signal, shut down workers gracefully'
+          opt :term_immediate,     'On TERM signal, shut down workers immediately (default)'
+          opt :single_process_group, 'Workers remain in the same process group as the master', default: false
+          opt :cluster, 'Name of the cluster this resque-pool belongs to', type: String, short: '-C'
+          opt :global_config, 'Alternate path to the global config file', type: String, short: '-G'
+        end
+        if opts[:daemon]
+          opts[:stdout] ||= 'log/resque-pool.stdout.log'
+          opts[:stderr] ||= 'log/resque-pool.stderr.log'
+          opts[:pidfile] ||= 'tmp/pids/resque-pool.pid'
+        end
+        opts
+      end
+      # rubocop:enable all
+    end
+  end
+end

data/lib/resque/pool/patches.rb

@@ -0,0 +1,40 @@
+require 'resque/cluster'
+
+module Resque
+  # Resque Pool monkey patched methods for resque-pool
+  class Pool
+    # add the running pool to distributed pool in order to manipulate it
+    def self.run
+      if GC.respond_to?(:copy_on_write_friendly=)
+        GC.copy_on_write_friendly = true
+      end
+      pool_config = Resque::Cluster.config ? {} : choose_config_file
+      started_pool = Resque::Pool.new(pool_config).start
+      Resque::Cluster.init(started_pool) if Resque::Cluster.config
+      started_pool.join
+      Resque::Cluster.member.unregister if Resque::Cluster.member
+    end
+
+    # performed inside the run loop, must check for any distributed pool updates
+    original_maintain_worker_count = instance_method(:maintain_worker_count)
+    define_method(:maintain_worker_count) do
+      cluster_update
+      original_maintain_worker_count.bind(self).call
+    end
+
+    def cluster_update
+      Resque::Cluster.member.perform if Resque::Cluster.member
+    end
+
+    def adjust_worker_counts(worker, number)
+      over_adjustment = ''
+      if @config[worker].to_i + number < 0
+        over_adjustment = "#{worker}:#{@config[worker].to_i + number}"
+        @config[worker] = 0
+      else
+        @config[worker] = @config[worker].to_i + number
+      end
+      over_adjustment
+    end
+  end
+end

data/resque-cluster.gemspec

@@ -0,0 +1,33 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'resque/cluster/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'resque-cluster'
+  s.version     = Resque::Cluster::VERSION
+  s.date        = '2015-07-23'
+  s.summary     = %q{Creates and manages resque worker in a distributed cluster}
+  s.description = %q{A management tool for resque workers. Allows spinning up and managing resque workers across multiple machines sharing the same Redis server}
+  s.authors     = ["Yasha Portnoy"]
+  s.email       = 'yash.portnoy@gmail.com'
+  s.homepage    = 'https://github.com/yportnoy/resque-cluster'
+  s.license     = 'MIT'
+
+  s.files         = `git ls-files -z`.split("\x0")
+  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ["lib"]
+
+  s.add_dependency  'resque-pool', '~> 0.5.0'
+  s.add_dependency  'gru', '0.0.3'
+
+  s.add_development_dependency 'pry', '> 0.0'
+  s.add_development_dependency 'awesome_print', '> 0.0'
+  s.add_development_dependency 'rspec', '~> 3.1.0'
+  s.add_development_dependency 'rdoc', '~> 3.12'
+  s.add_development_dependency 'bundler', '~> 1.7'
+  s.add_development_dependency 'jeweler', '~> 2.0.1'
+  s.add_development_dependency 'simplecov', '>= 0'
+  s.add_development_dependency 'rubocop', '~> 0.31'
+  s.add_development_dependency 'mock_redis', '~> 0.15.0'
+end

data/spec/global_config.yml

@@ -0,0 +1,3 @@
+foo: 2
+bar: 50
+"foo,bar,baz": 1

data/spec/global_rebalance_config.yml

@@ -0,0 +1,5 @@
+global_maximums:
+  foo: 2
+  bar: 50
+  "foo,bar,baz": 1
+rebalance_cluster: true

data/spec/integration/bin/resque-cluster_member_test

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+# -*- encoding: utf-8 -*-
+
+gem 'resque-pool'
+
+require 'resque/pool/cli'
+
+$LOAD_PATH.unshift File.expand_path('../../../../lib', __FILE__)
+
+require 'resque/pool/patches'
+require 'resque/pool/cli_patches'
+require 'resque/cluster'
+
+gem 'gru'
+
+module Gru
+  module Adapters
+    class RedisAdapter
+      def hostname
+        @hostname ||= ENV['GRU_HOSTNAME']
+      end
+    end
+  end
+end
+
+module Resque
+  class Cluster
+    class Member
+      private
+      def hostname
+        @hostname ||= ENV['GRU_HOSTNAME']
+      end
+    end
+  end
+end
+
+Resque::Pool::CLI.run

data/spec/integration/cluster_spec.rb

@@ -0,0 +1,181 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+LOCAL_CONFIG = "spec/integration/config/local_config.yml"
+GLOBAL_CONFIG = "spec/integration/config/global_config.yml"
+LOCAL_CONFIG2 = "spec/integration/config/local_config2.yml"
+GLOBAL_CONFIG2 = "spec/integration/config/global_config2.yml"
+GLOBAL_REBALANCE_CONFIG2 = "spec/integration/config/global_rebalance_config2.yml"
+
+RSpec.describe "Resque test-cluster" do
+  context "Spin Up and Down" do
+    before :all do
+      @a = TestMemberManager.new(LOCAL_CONFIG, GLOBAL_CONFIG)
+      @b = TestMemberManager.new(LOCAL_CONFIG, GLOBAL_CONFIG)
+      @c = TestMemberManager.new(LOCAL_CONFIG, GLOBAL_CONFIG)
+    end
+
+    it 'expects no workers to be running' do
+      expect(TestMemberManager.counts).to be_empty
+      expect(@a.counts).to be_empty
+      expect(@b.counts).to be_empty
+      expect(@c.counts).to be_empty
+    end
+
+    it 'expects counts to be correct after workers get spun up' do
+      @a.start
+      @b.start
+      @c.start
+      expect(TestMemberManager.counts).to eq({"par"=>2, "tar"=>8, "par,tar,var"=>1})
+    end
+
+    it 'cluster adjusts correctly when a member stops' do
+      @a.stop
+      expect(TestMemberManager.counts).to eq({"tar"=>6, "par"=>2, "par,tar,var"=>1})
+      expect(@a.counts).to be_empty
+      @b.stop
+      expect(TestMemberManager.counts).to eq({"tar"=>3, "par"=>1, "par,tar,var"=>1})
+      expect(@b.counts).to be_empty
+      @c.stop
+    end
+
+    after :all do
+      TestMemberManager.stop_all
+    end
+
+  end
+
+  context "Cluster with Rebalancing" do
+    before :all do
+      @d = TestMemberManager.new(LOCAL_CONFIG2, GLOBAL_REBALANCE_CONFIG2)
+      @e = TestMemberManager.new(LOCAL_CONFIG2, GLOBAL_REBALANCE_CONFIG2)
+      @f = TestMemberManager.new(LOCAL_CONFIG2, GLOBAL_REBALANCE_CONFIG2)
+      @d.start
+      @e.start
+      @f.start
+      sleep(5) # rebalance time
+    end
+
+    it 'expects counts to be correct after workers get spun up' do
+      expect(TestMemberManager.counts).to eq({"star"=>12})
+      expect(@d.counts).to eq({"star"=>4})
+      expect(@e.counts).to eq({"star"=>4})
+      expect(@f.counts).to eq({"star"=>4})
+    end
+
+    it 'adjusts correctly when a member stops' do
+      @d.stop
+      expect(TestMemberManager.counts).to eq({"star"=>12})
+      expect(@d.counts).to be_empty
+      expect(@e.counts).to eq({"star"=>6})
+      expect(@f.counts).to eq({"star"=>6})
+      @e.stop
+      @f.stop
+    end
+
+    after :all do
+      TestMemberManager.stop_all
+    end
+
+  end
+
+  context "Multiple Clusters and Environments" do
+    before :all do
+      @a = TestMemberManager.new(LOCAL_CONFIG, GLOBAL_CONFIG)
+      @b = TestMemberManager.new(LOCAL_CONFIG, GLOBAL_CONFIG, "test1-cluster")
+      @c = TestMemberManager.new(LOCAL_CONFIG, GLOBAL_CONFIG, "test-cluster", "test1")
+      @a.start
+      @b.start
+      @c.start
+      sleep(5) # rebalance time
+    end
+
+    it 'expects counts to be independent of each other' do
+      expect(TestMemberManager.counts).to eq({"tar"=>9, "par"=>3, "par,tar,var"=>3})
+      expect(@a.counts).to eq({"tar"=>3, "par"=>1, "par,tar,var"=>1})
+      expect(@b.counts).to eq({"tar"=>3, "par"=>1, "par,tar,var"=>1})
+      expect(@c.counts).to eq({"tar"=>3, "par"=>1, "par,tar,var"=>1})
+    end
+
+    after :all do
+      TestMemberManager.stop_all
+    end
+  end
+
+  context "Multiple Configs in the same cluster" do
+    before :all do
+      @a = TestMemberManager.new(LOCAL_CONFIG, GLOBAL_CONFIG)
+      @b = TestMemberManager.new(LOCAL_CONFIG2, GLOBAL_CONFIG)
+      @c = TestMemberManager.new(LOCAL_CONFIG, GLOBAL_REBALANCE_CONFIG2)
+      @a.start
+      @b.start
+      sleep(3) # rebalance time
+    end
+
+    it 'expects to have each cluster member only running workers in it\'s config' do
+      expect(TestMemberManager.counts).to eq({"tar"=>3, "par"=>1, "par,tar,var"=>1})
+      expect(@a.counts).to eq({"tar"=>3, "par"=>1, "par,tar,var"=>1})
+      expect(@b.counts).to be_empty
+    end
+
+    it 'expects the cluster to redistribute correctly after global config change' do
+      @c.start
+      sleep(8) # rebalance time
+      expect(TestMemberManager.counts).to eq({"star"=>4})
+      expect(@a.counts).to be_empty
+      expect(@b.counts).to eq({"star"=>4})
+      expect(@c.counts).to be_empty
+    end
+
+    after :all do
+      TestMemberManager.stop_all
+    end
+  end
+
+  context "Rebalance and non rebalance global configs switching in a cluster" do
+    before :all do
+      @a = TestMemberManager.new(LOCAL_CONFIG2, GLOBAL_REBALANCE_CONFIG2)
+      @b = TestMemberManager.new(LOCAL_CONFIG2, GLOBAL_CONFIG2)
+      @c = TestMemberManager.new(LOCAL_CONFIG2, GLOBAL_REBALANCE_CONFIG2)
+      @a.start
+      @b.start
+      @c.start
+      sleep(5) # rebalance time
+    end
+
+    it 'expects to have a correct number of workers in the cluster after multiple restarts' do
+      expect(TestMemberManager.counts).to eq({"star"=>12})
+      2.times do
+        sleep(5)
+        @a.stop
+        @a.start
+        sleep(5)
+        @b.stop
+        @b.start
+        sleep(5)
+        @c.stop
+        @c.start
+      end
+      sleep(8) # rebalance time
+      expect(TestMemberManager.counts).to eq({"star"=>12})
+      expect(@a.counts).to eq({"star"=>4})
+      expect(@b.counts).to eq({"star"=>4})
+      expect(@c.counts).to eq({"star"=>4})
+    end
+
+    it 'will not rebalance after the cluster is switched to rebalance-cluster false' do
+      @b.stop
+      sleep(2)
+      @b.start
+      sleep(8)
+      expect(TestMemberManager.counts).to eq({"star"=>12})
+      expect(@a.counts).to eq({"star"=>6})
+      expect(@b.counts).to eq({})
+      expect(@c.counts).to eq({"star"=>6})
+    end
+
+    after :all do
+      TestMemberManager.stop_all
+    end
+  end
+
+end

data/spec/integration/config/global_config.yml

@@ -0,0 +1,3 @@
+par: 2
+tar: 8
+"par,tar,var": 1

data/spec/integration/config/global_config2.yml

@@ -0,0 +1,3 @@
+global_maximums:
+  star: 12
+rebalance_cluster: false

data/spec/integration/config/global_rebalance_config2.yml

@@ -0,0 +1,3 @@
+global_maximums:
+  star: 12
+rebalance_cluster: true

data/spec/integration/config/local_config.yml

@@ -0,0 +1,3 @@
+par: 1
+tar: 3
+"par,tar,var": 1

data/spec/integration/config/local_config2.yml

@@ -0,0 +1 @@
+star: 6

data/spec/integration/spec_helper.rb

@@ -0,0 +1,3 @@
+require 'pry'
+require 'rspec'
+require File.expand_path(File.dirname(__FILE__) + '/test_member_manager')

data/spec/integration/standalone_spec.rb

@@ -0,0 +1,38 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+LOCAL_CONFIG = "spec/integration/config/local_config.yml"
+GLOBAL_CONFIG = "spec/integration/config/global_config.yml"
+
+RSpec.describe "resque-cluster" do
+  context "running 3 resque-cluster members in a standalone mode" do
+    before :all do
+      @a = TestMemberManager.new(LOCAL_CONFIG, GLOBAL_CONFIG, nil)
+      @b = TestMemberManager.new(LOCAL_CONFIG, GLOBAL_CONFIG, nil)
+      @c = TestMemberManager.new(LOCAL_CONFIG, GLOBAL_CONFIG, nil)
+    end
+
+    it 'expects no workers to be running' do
+      expect(@a.counts).to be_empty
+      expect(@b.counts).to be_empty
+      expect(@c.counts).to be_empty
+    end
+
+    it 'expects total counts to be correct after workers get spun up' do
+      @a.start
+      @b.start
+      @c.start
+      sleep(5)
+      expect(TestMemberManager.counts).to eq({"par"=>3, "tar"=>9, "par,tar,var"=>3})
+    end
+
+    it 'expects each resque-pool to have the same counts' do
+      expect(@a.counts).to eq({"par"=>1, "tar"=>3, "par,tar,var"=>1})
+      expect(@b.counts).to eq({"par"=>1, "tar"=>3, "par,tar,var"=>1})
+      expect(@c.counts).to eq({"par"=>1, "tar"=>3, "par,tar,var"=>1})
+    end
+
+    after :all do
+      TestMemberManager.stop_all
+    end
+  end
+end