resque-cluster 0.1.1.1 → 0.2.0

data/lib/resque/cluster/version.rb

@@ -1,5 +1,5 @@
 module Resque
   class Cluster
-    VERSION = '0.1.1.1'
+    VERSION = '0.2.0'
   end
 end

data/lib/resque/pool/patches.rb

@@ -22,6 +22,11 @@ module Resque
       original_maintain_worker_count.bind(self).call
     end
 
+    def quit
+      log "Quiting ..."
+      Process.kill(:TERM, Process.pid)
+    end
+
     def cluster_update
       Resque::Cluster.member.perform if Resque::Cluster.member
     end

data/resque-cluster.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_dependency  'resque-pool', '~> 0.5.0'
-  s.add_dependency  'gru', '0.1.1'
+  s.add_dependency  'gru', '0.1.3'
 
   s.add_development_dependency 'pry', '> 0.0'
   s.add_development_dependency 'awesome_print', '> 0.0'

data/spec/integration/cluster_spec.rb

@@ -2,9 +2,13 @@ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 
 LOCAL_CONFIG = "spec/integration/config/local_config.yml"
 GLOBAL_CONFIG = "spec/integration/config/global_config.yml"
+BAD_GLOBAL_CONFIG = "spec/integration/config/bad_global_config.yml"
+BAD_LOCAL_CONFIG = "spec/integration/config/bad_local_config.yml"
 LOCAL_CONFIG2 = "spec/integration/config/local_config2.yml"
 GLOBAL_CONFIG2 = "spec/integration/config/global_config2.yml"
 GLOBAL_REBALANCE_CONFIG2 = "spec/integration/config/global_rebalance_config2.yml"
+GLOBAL_REBALANCE_CONFIG3 = "spec/integration/config/global_rebalance_config3.yml"
+REBALANCE_CONFIG = "spec/integration/config/rebalance_config.yml"
 GLOBAL_CONFIG3 = "spec/integration/config/global_config3.yml"
 GLOBAL_CONFIG4 = "spec/integration/config/global_config4.yml"
 
@@ -49,6 +53,32 @@ RSpec.describe "Resque test-cluster" do
 
   end
 
+  context "Cluster with bad configs" do
+    before :all do
+      @d = TestMemberManager.new(LOCAL_CONFIG2, BAD_GLOBAL_CONFIG)
+      @e = TestMemberManager.new(BAD_LOCAL_CONFIG, GLOBAL_REBALANCE_CONFIG2)
+      @f = TestMemberManager.new("", "")
+      @d.start
+      @e.start
+      @f.start
+      sleep(5) # rebalance time
+    end
+
+    it 'expects counts to be correct after workers get spun up' do
+      expect(TestMemberManager.counts).to eq({})
+      expect(@d.is_running?).to eq(false)
+      expect(@e.is_running?).to eq(false)
+      expect(@f.is_running?).to eq(false)
+      expect(TestMemberManager.resque_cluster_members.count).to eq(0)
+    end
+
+    after :all do
+      TestMemberManager.stop_all
+    end
+
+  end
+
+
   context "Cluster with Rebalancing" do
     before :all do
       @d = TestMemberManager.new(LOCAL_CONFIG2, GLOBAL_REBALANCE_CONFIG2)
@@ -106,29 +136,29 @@ RSpec.describe "Resque test-cluster" do
     end
   end
 
-  context "Multiple Configs in the same cluster" do
+  context "Rebalance after global maximums change" do
     before :all do
       @a = TestMemberManager.new(LOCAL_CONFIG, GLOBAL_CONFIG)
-      @b = TestMemberManager.new(LOCAL_CONFIG2, GLOBAL_CONFIG)
-      @c = TestMemberManager.new(LOCAL_CONFIG, GLOBAL_REBALANCE_CONFIG2)
+      @b = TestMemberManager.new(LOCAL_CONFIG, GLOBAL_CONFIG)
+      @c = TestMemberManager.new(LOCAL_CONFIG, GLOBAL_REBALANCE_CONFIG3)
       @a.start
       @b.start
       sleep(3) # rebalance time
     end
 
-    it 'expects to have each cluster member only running workers in it\'s config' do
-      expect(TestMemberManager.counts).to eq({"tar"=>3, "par"=>1, "par,tar,var"=>1})
-      expect(@a.counts).to eq({"tar"=>3, "par"=>1, "par,tar,var"=>1})
-      expect(@b.counts).to be_empty
+    it 'expects to have each cluster member only running workers in its config' do
+      expect(TestMemberManager.counts).to eq('par' => 2, 'tar' => 6, 'par,tar,var' => 1)
+      expect(@a.counts).to eq('par' => 1, 'tar' => 3, 'par,tar,var' => 1)
+      expect(@b.counts).to eq('par' => 1, 'tar' => 3)
     end
 
     it 'expects the cluster to redistribute correctly after global config change' do
       @c.start
       sleep(8) # rebalance time
-      expect(TestMemberManager.counts).to eq({"star"=>4})
-      expect(@a.counts).to be_empty
-      expect(@b.counts).to eq({"star"=>4})
-      expect(@c.counts).to be_empty
+      expect(TestMemberManager.counts).to eq('par' => 3, 'tar' => 6, 'par,tar,var' => 2)
+      expect(@a.counts).to eq('par' => 1, 'tar' => 2, 'par,tar,var' => 1)
+      expect(@b.counts).to eq('par' => 1, 'tar' => 2)
+      expect(@c.counts).to eq('par' => 1, 'tar' => 2, 'par,tar,var' => 1)
     end
 
     after :all do

data/spec/integration/config/bad_global_config.yml

@@ -0,0 +1,3 @@
+par= 2
+? tar: 8
+"par,tar,var"+ 1

data/spec/integration/config/global_rebalance_config3.yml

@@ -0,0 +1,5 @@
+rebalance_cluster: true
+global_maximums:
+  par: 4
+  tar: 6
+  "par,tar,var": 2

data/spec/integration/config/rebalance_config.yml

@@ -0,0 +1,4 @@
+par: 4
+tar: 4
+"par,tar,var": 2
+star: 8

data/spec/integration/spec_helper.rb

@@ -1,3 +1,6 @@
 require 'pry'
 require 'rspec'
+require 'sys/proctable'
+require 'resque'
+
 require File.expand_path(File.dirname(__FILE__) + '/test_member_manager')

data/spec/integration/test_member_manager.rb

@@ -1,4 +1,7 @@
 class TestMemberManager
+  include Sys
+
+  attr_accessor :pid
 
   def initialize(local_config_path, global_config_path, cluster_name = "test-cluster", environment = "test")
     @local_config_path = local_config_path
@@ -12,53 +15,71 @@ class TestMemberManager
   def start
     ENV['GRU_HOSTNAME'] = hostname
     @pid = spawn("bundle exec spec/integration/bin/resque-cluster_member_test -c #{@local_config_path} -E #{@environment}#{@cluster_name.nil? ? "" : " -C "+@cluster_name} -G #{@global_config_path}")
+    count = 0
 
-    while ( @pool_master_pid.nil? ) do
+    while ( @pool_master_pid.nil? && count <= 100 ) do
       sleep(0.1)
-      child_process = @pid #`pgrep -P #{@pid}`.strip
-      pool = `ps -p #{child_process} -hf | grep 'resque-pool-master\\[resque-cluster\\]: managing \\[' | awk '{print $1}'`.strip.to_i
-      @pool_master_pid = pool > 0 ? pool : nil
+
+      if (ProcTable.ps(@pid) &&
+          ProcTable.ps(@pid).cmdline =~ /resque-pool-master\[resque-cluster\]:\smanaging\s\[/)
+        pool_pid = ProcTable.ps(@pid).pid
+      end
+
+      @pool_master_pid = pool_pid ? pool_pid : nil
+      count += 1
     end
-    puts "Pool Master pid is ---------- #{@pool_master_pid}"
+
+    puts "Pool Master pid is ---------- #{@pool_master_pid}" if @pool_master_pid
   end
 
   def stop
     puts "************************************************ About to kill : Pool Master pid ---------- #{@pool_master_pid}"
-    Process.kill(:TERM, @pool_master_pid)
+    Process.kill(:QUIT, @pool_master_pid)
     while ( @pool_master_pid ) do
-      pool = `ps -p #{@pool_master_pid} -hf | grep 'resque-pool-master\\[resque-cluster\\]: managing \\[' | awk '{print $1}'`.strip.to_i
-      @pool_master_pid = pool > 0 ? pool : nil
+      if (ProcTable.ps(@pool_master_pid) &&
+          ! (ProcTable.ps(@pool_master_pid).cmdline =~ /resque-pool-master\[resque-cluster\]:\smanaging\s\[/))
+        @pool_master_pid = nil
+      end
     end
     @pid = nil
-    sleep(5)
+    sleep(3)
+  end
+
+  def is_running?
+    (ProcTable.ps(@pid).instance_of? (Struct::ProcTableStruct)) &&
+      ! (ProcTable.ps(@pid).cmdline =~ /resque-pool-master\[resque-cluster\]:\smanaging\s\[/).nil?
   end
 
   def kill
     puts "************************************************ About to kill -9 : Pool Master pid ---------- #{@pool_master_pid}"
-    `kill -9 #{@pool_master_pid}`
+    Process.kill(:TERM, @pool_master_pid)
     while ( @pool_master_pid ) do
-      pool = `ps -p #{@pool_master_pid} -hf | grep 'resque-pool-master\\[resque-cluster\\]: managing \\[' | awk '{print $1}'`.strip.to_i
-      @pool_master_pid = pool > 0 ? pool : nil
+      if (ProcTable.ps(@pool_master_pid) &&
+          ! (ProcTable.ps(@pool_master_pid).cmdline =~ /resque-pool-master\[resque-cluster\]:\smanaging\s\[/))
+        @pool_master_pid = nil
+      end
     end
     @pid = nil
-    sleep(5)
+    sleep(3)
   end
 
   def counts
     return {} unless @pool_master_pid
-    local_workers = `ps --ppid #{@pool_master_pid} -fh | awk '{print $8}'`.split
+    local_workers = ProcTable.ps.select{|p| p.ppid == @pool_master_pid}.map(&:cmdline)
     TestMemberManager.worker_counts(local_workers)
   end
 
   def self.counts
-    all_workers = `ps -ef | grep "resque-" | grep "Waiting for" | grep -v ps| awk '{print $11}'`.split
+    pool_pids = resque_cluster_members.map(&:pid)
+    all_workers = ProcTable.ps.select{|p| pool_pids.include? p.ppid}.map(&:cmdline)
     worker_counts(all_workers)
   end
 
   def self.worker_counts(worker_array)
     final_counts = Hash.new(0)
 
-    worker_array.each do |worker|
+    worker_array.each do |worker_cmdline|
+      worker = parse_worker_name(worker_cmdline)
       final_counts[worker] += 1
     end
 
@@ -69,13 +90,23 @@ class TestMemberManager
     @hostname ||= "#{Socket.gethostname}-#{member_count+1}"
   end
 
+  def self.parse_worker_name(worker_cmdline)
+    worker_cmdline.gsub(/resque(\d|\.|-)*:\sWaiting\sfor\s/, '')
+  end
+
   def self.stop_all
-    pools = `ps -ef | grep 'resque-pool-master\\[resque-cluster\\]: managing \\[' | awk '{print $2}'`.split
-    `kill #{pools.join(' ')}` unless pools.empty?
+    TestMemberManager.resque_cluster_members.each do |member|
+      Process.kill(:TERM, member.pid)
+    end
     sleep(3)
   end
 
   def member_count
-    `ps -ef | grep resque-pool-master | grep -v grep|wc -l`.strip.to_i
+    TestMemberManager.resque_cluster_members.count
+  end
+
+  def self.resque_cluster_members
+    ProcTable.ps.select{|p| p.cmdline =~ /resque-pool-master/}
   end
+
 end

data/spec/unit/config/verifier_spec.rb

@@ -0,0 +1,86 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+module Resque
+  RSpec.describe Cluster::Config::Verifier do
+    let(:valid_local_config)  { Cluster::Config::File.new(support_dir + 'valid_local_config.yml') }
+    let(:valid_global_config) { Cluster::Config::File.new(support_dir + 'valid_global_config.yml') }
+    let(:broken_yaml_config)  { Cluster::Config::File.new(support_dir + 'broken_yaml_config.yml') }
+    let(:non_hash_config)     { Cluster::Config::File.new(support_dir + 'non_hash_config.yml') }
+    let(:empty_config)        { Cluster::Config::File.new(support_dir + 'empty_config.yml') }
+    let(:missing_config)      { Cluster::Config::File.new(support_dir + 'missing') }
+
+    subject { Cluster::Config::Verifier.new(configs) }
+
+    shared_examples_for 'valid config' do
+      it 'verifies' do
+        expect(subject).to be_verified
+      end
+    end
+
+    shared_examples_for 'invalid config' do
+      it "doesn't verify" do
+        expect(subject).not_to be_verified
+      end
+    end
+
+    context 'with valid configs' do
+      let(:configs) { [valid_local_config, valid_global_config] }
+
+      it_behaves_like 'valid config'
+    end
+
+    context 'with a single config file' do
+      let(:configs) { [valid_local_config] }
+
+      it_behaves_like 'valid config'
+    end
+
+    context 'with a missing config' do
+      let(:configs) { [missing_config] }
+
+      it_behaves_like 'invalid config'
+
+      it 'reports the error' do
+        subject.verified?
+
+        expect(subject.configs.flat_map { |config| config.errors.to_a }).to include("Configuration file doesn't exist")
+      end
+    end
+
+    context 'with a non-hash config' do
+      let(:configs) { [non_hash_config] }
+
+      it_behaves_like 'invalid config'
+
+      it 'reports the error' do
+        subject.verified?
+
+        expect(subject.configs.flat_map { |config| config.errors.to_a }).to include("Parsed config as invalid type: expected Hash, got FalseClass")
+      end
+    end
+
+    context 'with an empty config' do
+      let(:configs) { [empty_config] }
+
+      it_behaves_like 'invalid config'
+
+      it 'reports the error' do
+        subject.verified?
+
+        expect(subject.configs.flat_map { |config| config.errors.to_a }).to include("Config file is empty")
+      end
+    end
+
+    context 'with broken YAML' do
+      let(:configs) { [broken_yaml_config] }
+
+      it_behaves_like 'invalid config'
+
+      it 'reports the error' do
+        subject.verified?
+
+        expect(subject.configs.flat_map { |config| config.errors.to_a }).to include('(<unknown>): did not find expected comment or line break while scanning a block scalar at line 1 column 1')
+      end
+    end
+  end
+end

data/spec/unit/config_spec.rb

@@ -0,0 +1,194 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+RSpec.describe Resque::Cluster::Config do
+  let(:redis) { Resque.redis }
+
+  before do
+    config.verified?
+
+    Resque::Cluster.config = {
+      cluster_name: 'unit-test-cluster',
+      environment: 'unit-test'
+    }
+  end
+
+  shared_examples_for 'a valid config' do
+    it 'is verified' do
+      expect(config).to be_verified
+    end
+
+    it "gru_format should return a correct hash" do
+      expect(config.gru_format).to eql(correct_hash)
+    end
+  end
+
+  describe 'with a single config file' do
+    let(:config) { Resque::Cluster::Config.new(config_path) }
+    let(:correct_hash) do
+      {
+        cluster_maximums:         { 'foo' => 2, 'bar' => 50, "foo,bar,baz" => 1 },
+        host_maximums:            { 'foo' => 1, 'bar' => 9, "foo,bar,baz" => 1 },
+        client_settings:          redis.client.options,
+        rebalance_flag:           true,
+        presume_host_dead_after:  60,
+        max_workers_per_host:     10,
+        cluster_name:             "unit-test-cluster",
+        environment_name:         "unit-test",
+        manage_worker_heartbeats: true,
+        version_hash:             `git rev-parse --verify HEAD`.strip
+      }
+    end
+
+    context 'old style' do
+      let(:config_path) { support_dir + 'valid_single_old_config.yml' }
+
+      it_behaves_like 'a valid config'
+
+      context 'with a missing local maximum' do
+        let(:config) { Resque::Cluster::Config.new(config_path) }
+        let(:config_path) { support_dir + 'single_old_missing_local.yml' }
+
+        it 'should not be verified' do
+          expect(config).not_to be_verified
+        end
+
+        it 'should not be verified' do
+          expect(config.errors).to contain_exactly("Every worker configuration must contain a local and a global maximum.")
+        end
+      end
+
+      context 'with a missing global maximum' do
+        let(:config) { Resque::Cluster::Config.new(config_path) }
+        let(:config_path) { support_dir + 'single_old_missing_global.yml' }
+
+        it 'should not be verified' do
+          expect(config).not_to be_verified
+        end
+
+        it 'should not be verified' do
+          expect(config.errors).to contain_exactly("Every worker configuration must contain a local and a global maximum.")
+        end
+      end
+    end
+
+    context 'new style' do
+      let(:config_path) { support_dir + 'valid_single_new_config.yml' }
+
+      it_behaves_like 'a valid config'
+
+      context 'with a missing local maximum' do
+        let(:config) { Resque::Cluster::Config.new(config_path) }
+        let(:config_path) { support_dir + 'missing_local_maximum.yml' }
+
+        it 'should not be verified' do
+          expect(config).not_to be_verified
+        end
+
+        it 'should not be verified' do
+          expect(config.errors).to contain_exactly("Every worker configuration must contain a local and a global maximum.")
+        end
+      end
+
+      context 'with a missing global maximum' do
+        let(:config) { Resque::Cluster::Config.new(config_path) }
+        let(:config_path) { support_dir + 'missing_global_maximum.yml' }
+
+        it 'should not be verified' do
+          expect(config).not_to be_verified
+        end
+
+        it 'should not be verified' do
+          expect(config.errors).to contain_exactly("Every worker configuration must contain a local and a global maximum.")
+        end
+      end
+    end
+  end
+
+  describe 'separate config files' do
+    context "with valid config files" do
+      let(:config)             { Resque::Cluster::Config.new(local_config_path, global_config_path) }
+      let(:local_config_path)  { File.expand_path(File.dirname(__FILE__) + '/../local_config.yml') }
+      let(:global_config_path) { File.expand_path(File.dirname(__FILE__) + '/../global_config.yml') }
+
+      let(:correct_hash) do
+        {
+          cluster_maximums:         { 'foo' => 2, 'bar' => 50, "foo,bar,baz" => 1 },
+          host_maximums:            { 'foo' => 1, 'bar' => 9, "foo,bar,baz" => 1 },
+          client_settings:          redis.client.options,
+          rebalance_flag:           false,
+          presume_host_dead_after:  120,
+          max_workers_per_host:     nil,
+          cluster_name:             "unit-test-cluster",
+          environment_name:         "unit-test",
+          manage_worker_heartbeats: true,
+          version_hash:             `git rev-parse --verify HEAD`.strip
+        }
+      end
+
+      it_behaves_like 'a valid config'
+
+      it "config should have no warnings or errors" do
+        expect(config.errors.count).to eql(0)
+        expect(config.warnings.count).to eql(0)
+      end
+
+      it "git_version_hash should be set" do
+        expect(config.version_git_hash).to eql(`git rev-parse --verify HEAD`.strip)
+      end
+    end
+
+    context 'with a missing local maximum' do
+      let(:config)             { Resque::Cluster::Config.new(local_config_path, global_config_path) }
+      let(:local_config_path)  { support_dir + 'separate_missing_local.yml' }
+      let(:global_config_path) { File.expand_path(File.dirname(__FILE__) + '/../global_config.yml') }
+
+      it 'should not be verified' do
+        expect(config).not_to be_verified
+      end
+
+      it 'should not be verified' do
+        expect(config.errors).to contain_exactly("Every worker configuration must contain a local and a global maximum.")
+      end
+    end
+
+    context 'with a missing global maximum' do
+      let(:config)             { Resque::Cluster::Config.new(local_config_path, global_config_path) }
+      let(:local_config_path)  { File.expand_path(File.dirname(__FILE__) + '/../local_config.yml') }
+      let(:global_config_path) { support_dir + 'separate_missing_global.yml' }
+
+      it 'should not be verified' do
+        expect(config).not_to be_verified
+      end
+
+      it 'should not be verified' do
+        expect(config.errors).to contain_exactly("Every worker configuration must contain a local and a global maximum.")
+      end
+    end
+
+    context "with invalid config file" do
+      let(:local_config_path)  { File.expand_path(File.dirname(__FILE__) + '/../local_configuration.yml') }
+      let(:global_config_path) { File.expand_path(File.dirname(__FILE__) + '/../../README.rdoc') }
+      let(:config)             { Resque::Cluster::Config.new(local_config_path, global_config_path) }
+      let(:correct_hash)       { {} }
+
+      it "should not be verified" do
+        expect(config.verified?).to eql(false)
+      end
+
+      it "config should have no warnings but 2 errors" do
+        expect(config.errors.count).to eql(1)
+        expect(config.warnings.count).to eql(0)
+        expect(config.errors).to contain_exactly(
+          "#{local_config_path}: Configuration file doesn't exist")
+      end
+
+      it "gru_format should return a an empty hash" do
+        expect(config.gru_format).to eql(correct_hash)
+      end
+
+      it "git_version_hash should not be set" do
+        expect(config.version_git_hash).to be_nil
+      end
+    end
+  end
+end