gru 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 89ae1e251f9db9637786d27b5abeb7de7fd0bef7
-  data.tar.gz: 0dbd72c564a5622ec09abd483d408c7c851e072b
+  metadata.gz: c8569b8dac503cf08983a463bb293a277f09f301
+  data.tar.gz: 45da184cdbe70af5c0f871d65a9d4f60de2555f6
 SHA512:
-  metadata.gz: b3740b2ee29bc4f65968eedd061d5f6fb506a24152088329e7297ecc519b67f8f1f044b03fe515a185987a09691b7b0bf564a9fa0cc2801adca8072df8fd2796
-  data.tar.gz: f360078d5e374287902e5ce48d78cd6015dd7a83bf986b993ad5a771f2dda556cc64335848f48f81e2dbfe800b3f15f2af04255b013b5e9ff5efc30e4774384c
+  metadata.gz: ad6bc8c502243e5f9f8ce86dfe6275758f4eff9ad55f562d070ad99484782f58d1bb49730413d3314e42c5dd92b8fb6f4950a9c290fb97d1168692fd168a264a
+  data.tar.gz: 21d48ee74d9fc7bda158b7f2485bbb97ba35b6e9401bbc2a306070a802262f8ae7a78c65df7fdb0d552d6e013cdffcedd95989cd61719e59b10c31cdab60fea8

data/lib/gru/adapters/redis_adapter.rb

@@ -190,24 +190,24 @@ module Gru
         host_running,global_running,host_max,global_max = worker_counts(worker)
         result = false
         if rebalance_cluster?
-          result = host_running.to_i < max_workers_per_host(global_max,host_max) &&
-            host_running.to_i < @settings.max_worker_processes_per_host
+          result = host_running.to_i < max_workers_per_host(global_max,host_max)
         else
           result = host_running.to_i < host_max.to_i
         end
-          result && global_running.to_i < global_max.to_i
+        result && global_running.to_i < global_max.to_i &&
+          total_host_running < @settings.max_worker_processes_per_host
       end
 
       def expire_worker?(worker)
         host_running,global_running,host_max,global_max = worker_counts(worker)
         result = false
         if rebalance_cluster?
-          result = host_running.to_i > max_workers_per_host(global_max,host_max) ||
-            host_running.to_i > @settings.max_worker_processes_per_host
+          result = host_running.to_i > max_workers_per_host(global_max,host_max)
         else
           result = host_running.to_i > host_max.to_i
         end
-         (result || global_running.to_i > global_max.to_i) && host_running.to_i >= 0
+         (result || global_running.to_i > global_max.to_i) && host_running.to_i >= 0 ||
+           total_host_running > @settings.max_worker_processes_per_host
       end
 
       def worker_counts(worker)
@@ -232,6 +232,10 @@ module Gru
         send_message(:hget,host_workers_running_key,worker).to_i
       end
 
+      def total_host_running
+        send_message(:hgetall,host_workers_running_key).values.reduce(0) {|sum, value| sum + value.to_i }
+      end
+
       def gru_host_count
         send_message(:keys,"#{gru_key}:*:workers_running").count - 1
       end

data/lib/gru/version.rb

@@ -1,3 +1,3 @@
 module Gru
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/spec/gru/adapters/redis_adapter_spec.rb

@@ -84,6 +84,7 @@ describe Gru::Adapters::RedisAdapter do
         expect(client).to receive(:setnx).exactly(3).times.and_return(true)
         expect(client).to receive(:del).with("#{gru_key}:test_worker").exactly(3).times
         expect(client).to receive(:get).with("#{gru_key}:rebalance").exactly(3).times
+        expect(client).to receive(:hgetall).with("#{gru_key}:#{hostname}:workers_running").exactly(3).times.and_return({'test_worker' => '1'})
         expect(client).to receive(:hincrby).with("#{gru_key}:global:workers_running",'test_worker',1).exactly(3).times
         expect(client).to receive(:hincrby).with("#{gru_key}:#{hostname}:workers_running",'test_worker',1).exactly(3).times
         expect(client).to receive(:hget).with("#{gru_key}:#{hostname}:max_workers",'test_worker').exactly(2).times
@@ -99,6 +100,7 @@ describe Gru::Adapters::RedisAdapter do
         expect(client).to receive(:setnx).exactly(3).times.and_return(true)
         expect(client).to receive(:del).with("#{gru_key}:test_worker").exactly(3).times
         expect(client).to receive(:get).with("#{gru_key}:rebalance").exactly(3).times
+        expect(client).to receive(:hgetall).with("#{gru_key}:#{hostname}:workers_running").exactly(3).times.and_return({'test_worker' => '1'})
         expect(client).to receive(:hincrby).with("#{gru_key}:global:workers_running",'test_worker',1).exactly(3).times
         expect(client).to receive(:hincrby).with("#{gru_key}:#{hostname}:workers_running",'test_worker',1).exactly(3).times
         expect(client).to receive(:hget).with("#{gru_key}:#{hostname}:max_workers",'test_worker').exactly(2).times
@@ -189,11 +191,13 @@ describe Gru::Adapters::RedisAdapter do
 
     it "doesn't remove workers when local maximum has not been exceeded" do
       expect(client).to receive(:multi).exactly(1).times.and_yield(client).and_return([3,3,4,3])
+      expect(client).to receive(:hgetall).with("#{gru_key}:#{hostname}:workers_running").exactly(1).times.and_return({'test_worker' => '1'})
       expect(adapter.expire_workers).to eq({'test_worker' => 0})
     end
 
     it "doesn't remove workers when global maximum has not been exceeded" do
       expect(client).to receive(:multi).exactly(1).times.and_yield(client).and_return([3,4,3,5])
+      expect(client).to receive(:hgetall).with("#{gru_key}:#{hostname}:workers_running").exactly(1).times.and_return({'test_worker' => '1'})
       expect(adapter.expire_workers).to eq({'test_worker' => 0})
     end
   end
@@ -220,6 +224,7 @@ describe Gru::Adapters::RedisAdapter do
       expect(client).to receive(:hget).with("#{gru_key}:#{hostname}:max_workers",'test_worker').exactly(3).times
       expect(client).to receive(:hget).with("#{gru_key}:global:max_workers",'test_worker').exactly(3).times
       expect(client).to receive(:keys).with("#{gru_key}:*:workers_running").exactly(3).times.and_return(['foo'])
+      expect(client).to receive(:hgetall).with("#{gru_key}:#{hostname}:workers_running").exactly(3).times.and_return({'test_worker' => '1'})
       expect(client).to receive(:setnx).exactly(3).times.and_return(true)
       expect(client).to receive(:hincrby).with("#{gru_key}:global:workers_running",'test_worker',1).exactly(3).times
       expect(client).to receive(:hincrby).with("#{gru_key}:#{hostname}:workers_running",'test_worker',1).exactly(3).times
@@ -233,6 +238,7 @@ describe Gru::Adapters::RedisAdapter do
       expect(client).to receive(:hgetall).with("#{gru_key}:#{hostname}:max_workers").and_return(config.cluster_maximums)
       expect(client).to receive(:keys).with("#{gru_key}:*:workers_running").exactly(3).times.and_return(["test1","test2"])
       expect(client).to receive(:get).with("#{gru_key}:rebalance").exactly(3).times.and_return("true")
+      expect(client).to receive(:hgetall).with("#{gru_key}:#{hostname}:workers_running").exactly(3).times.and_return({'test_worker' => '10'})
       expect(client).to receive(:hget).with("#{gru_key}:#{hostname}:max_workers",'test_worker').exactly(3).times
       expect(client).to receive(:hget).with("#{gru_key}:global:max_workers",'test_worker').exactly(3).times
       expect(client).to receive(:hget).with("#{gru_key}:#{hostname}:workers_running",'test_worker').exactly(3).times
@@ -245,6 +251,8 @@ describe Gru::Adapters::RedisAdapter do
       expect(client).to receive(:multi).exactly(3).times.and_yield(client).and_return([9,20,20,30], [10,20,20,30])
       expect(client).to receive(:hgetall).with("#{gru_key}:#{hostname}:max_workers").and_return(config.cluster_maximums)
       expect(client).to receive(:setnx).exactly(1).times.and_return(true)
+      expect(client).to receive(:hgetall).with("#{gru_key}:#{hostname}:workers_running").exactly(3).times.and_return(
+        {'test_worker' => '9'}, {'test_worker' => '10'}, {'test_worker' => '10'})
       expect(client).to receive(:hincrby).with("#{gru_key}:foo:workers_running",'test_worker',1).exactly(1).times
       expect(client).to receive(:hincrby).with("#{gru_key}:global:workers_running",'test_worker',1).exactly(1).times
       expect(client).to receive(:del).with("#{gru_key}:test_worker").exactly(1).times

data/spec/gru/gru_integration_spec.rb

@@ -8,7 +8,6 @@ require 'pry'
 # This requires a redis server instance running on localhost
 # un-pend to run with actual redis-server
 xdescribe Gru do
-
   after {
     client.flushdb
   }
@@ -167,5 +166,323 @@ xdescribe Gru do
       test_client.hset("GRU:default:default:heartbeats", 'test3', Time.now - 300)
       expect(test2.adjust_workers).to eq( { 'test_worker' => 3 })
     end
+
+  end
+
+  context "max workers per host with high maximum worker count" do
+    let(:settings) {
+      {
+        rebalance_flag: true,
+        manage_worker_heartbeats: true,
+        max_workers_per_host: nil,
+        cluster_maximums: {
+        'test_worker' => '100'
+        }
+      }
+    }
+
+    let(:test_client) {
+      Redis.new
+    }
+
+    let(:adapter1) {
+      adapter1 = Gru::Adapters::RedisAdapter.new(Gru::Configuration.new(settings.clone))
+      allow(adapter1).to receive(:hostname).and_return('test1')
+      adapter1
+    }
+
+    let(:adapter2) {
+      adapter2 = Gru::Adapters::RedisAdapter.new(Gru::Configuration.new(settings.clone))
+      allow(adapter2).to receive(:hostname).and_return('test2')
+      adapter2
+    }
+
+    let(:adapter3) {
+      adapter3 = Gru::Adapters::RedisAdapter.new(Gru::Configuration.new(settings.clone))
+      allow(adapter3).to receive(:hostname).and_return('test3')
+      adapter3
+    }
+
+    let(:adapter4) {
+      adapter4 = Gru::Adapters::RedisAdapter.new(Gru::Configuration.new(settings.clone))
+      allow(adapter4).to receive(:hostname).and_return('test4')
+      adapter4
+    }
+
+    it "doesn't exceed max processes per host setting" do
+      test1 = Gru::WorkerManager.new(adapter1)
+      test2 = Gru::WorkerManager.new(adapter2)
+      test3 = Gru::WorkerManager.new(adapter3)
+      test1.register_workers
+      test2.register_workers
+      test3.register_workers
+      expect(test1.adjust_workers).to eq( { 'test_worker' => 30 })
+      expect(test2.adjust_workers).to eq( { 'test_worker' => 30 })
+      expect(test3.adjust_workers).to eq( { 'test_worker' => 30 })
+    end
+
+    it "honors the max processes per host setting and correctly rebalances worker counts" do
+      test1 = Gru::WorkerManager.new(adapter1)
+      test2 = Gru::WorkerManager.new(adapter2)
+      test3 = Gru::WorkerManager.new(adapter3)
+      test4 = Gru::WorkerManager.new(adapter4)
+      test1.register_workers
+      test2.register_workers
+      test3.register_workers
+      # Honors max processes per host setting
+      expect(test1.adjust_workers).to eq({ 'test_worker' => 30 })
+      expect(test2.adjust_workers).to eq({ 'test_worker' => 30 })
+      expect(test3.adjust_workers).to eq({ 'test_worker' => 30 })
+      test4.register_workers
+      # Pick up remaining workers
+      expect(test4.adjust_workers).to eq({ 'test_worker' => 10 })
+      # Rebalance workers across hosts
+      expect(test3.adjust_workers).to eq({ 'test_worker' => -5 })
+      expect(test2.adjust_workers).to eq({ 'test_worker' => -5 })
+      expect(test1.adjust_workers).to eq({ 'test_worker' => -5 })
+      expect(test4.adjust_workers).to eq({ 'test_worker' => 15 })
+      # No more workers to pick up on the hosts
+      expect(test4.adjust_workers).to eq({ 'test_worker' => 0 })
+      expect(test3.adjust_workers).to eq({ 'test_worker' => 0 })
+      expect(test2.adjust_workers).to eq({ 'test_worker' => 0 })
+      expect(test1.adjust_workers).to eq({ 'test_worker' => 0 })
+      expect(test4.adjust_workers).to eq({ 'test_worker' => 0 })
+      # Release workers
+      test4.release_workers
+      # Rebalance for lost host
+      expect(test1.adjust_workers).to eq({ 'test_worker' => 5 })
+      expect(test2.adjust_workers).to eq({ 'test_worker' => 5 })
+      expect(test3.adjust_workers).to eq({ 'test_worker' => 5 })
+      # No more workers due to max per host limit
+      expect(test1.adjust_workers).to eq({ 'test_worker' => 0 })
+      expect(test2.adjust_workers).to eq({ 'test_worker' => 0 })
+      expect(test3.adjust_workers).to eq({ 'test_worker' => 0 })
+    end
+
+  end
+
+  context "max workers per host with high maximum worker count" do
+    let(:settings) {
+      {
+        rebalance_flag: true,
+        manage_worker_heartbeats: true,
+        max_workers_per_host: 5,
+        cluster_maximums: {
+        'test_worker1' => '3',
+        'test_worker2' => '3',
+        'test_worker3' => '3',
+        'test_worker4' => '3',
+        'test_worker5' => '3',
+        'test_worker6' => '3'
+        }
+      }
+    }
+
+    let(:test_client) {
+      Redis.new
+    }
+
+    let(:adapter1) {
+      adapter1 = Gru::Adapters::RedisAdapter.new(Gru::Configuration.new(settings.clone))
+      allow(adapter1).to receive(:hostname).and_return('test1')
+      adapter1
+    }
+
+    let(:adapter2) {
+      adapter2 = Gru::Adapters::RedisAdapter.new(Gru::Configuration.new(settings.clone))
+      allow(adapter2).to receive(:hostname).and_return('test2')
+      adapter2
+    }
+
+    let(:adapter3) {
+      adapter3 = Gru::Adapters::RedisAdapter.new(Gru::Configuration.new(settings.clone))
+      allow(adapter3).to receive(:hostname).and_return('test3')
+      adapter3
+    }
+
+    let(:adapter4) {
+      adapter4 = Gru::Adapters::RedisAdapter.new(Gru::Configuration.new(settings.clone))
+      allow(adapter4).to receive(:hostname).and_return('test4')
+      adapter4
+    }
+
+    it "handles provisioning with many workers and few worker instances" do
+      test1 = Gru::WorkerManager.new(adapter1)
+      test2 = Gru::WorkerManager.new(adapter2)
+      test3 = Gru::WorkerManager.new(adapter3)
+      test4 = Gru::WorkerManager.new(adapter4)
+      test1.register_workers
+      test2.register_workers
+      test3.register_workers
+      # Honors max processes per host setting
+      expect(test1.adjust_workers).to eq({
+        'test_worker1' => 1,
+        'test_worker2' => 1,
+        'test_worker3' => 1,
+        'test_worker4' => 1,
+        'test_worker5' => 1,
+        'test_worker6' => 0
+      })
+
+      expect(test2.adjust_workers).to eq({
+        'test_worker1' => 1,
+        'test_worker2' => 1,
+        'test_worker3' => 1,
+        'test_worker4' => 1,
+        'test_worker5' => 1,
+        'test_worker6' => 0
+      })
+
+      expect(test3.adjust_workers).to eq({
+        'test_worker1' => 1,
+        'test_worker2' => 1,
+        'test_worker3' => 1,
+        'test_worker4' => 1,
+        'test_worker5' => 1,
+        'test_worker6' => 0
+      })
+      test4.register_workers
+      # Adds new worker based on new instance
+      expect(test4.adjust_workers).to eq({
+        'test_worker1' => 0,
+        'test_worker2' => 0,
+        'test_worker3' => 0,
+        'test_worker4' => 0,
+        'test_worker5' => 0,
+        'test_worker6' => 1
+      })
+
+      # Doesn't alter existing worker instances
+      # due to max_workers_per_host balance
+      expect(test1.adjust_workers).to eq({
+        'test_worker1' => 0,
+        'test_worker2' => 0,
+        'test_worker3' => 0,
+        'test_worker4' => 0,
+        'test_worker5' => 0,
+        'test_worker6' => 0
+      })
+    end
+
+    it "handles expiring with many workers and few worker instances" do
+      test1 = Gru::WorkerManager.new(adapter1)
+      test1.register_workers
+      args = ["GRU:default:default:test1:workers_running",
+              'test_worker1',1,'test_worker2',1,'test_worker3',1,
+              'test_worker4',1,'test_worker5',1,'test_worker6',1
+             ]
+      client.hmset(*args)
+
+      # Honors max processes per host setting
+      expect(test1.adjust_workers).to eq({
+        'test_worker1' => -1,
+        'test_worker2' => 0,
+        'test_worker3' => 0,
+        'test_worker4' => 0,
+        'test_worker5' => 0,
+        'test_worker6' => 0
+      })
+
+      expect(test1.adjust_workers).to eq({
+        'test_worker1' => 0,
+        'test_worker2' => 0,
+        'test_worker3' => 0,
+        'test_worker4' => 0,
+        'test_worker5' => 0,
+        'test_worker6' => 0
+      })
+    end
+  end
+  context "max workers per host without rebalance flag" do
+    let(:settings) {
+      {
+        rebalance_flag: false,
+        manage_worker_heartbeats: true,
+        max_workers_per_host: 5,
+        cluster_maximums: {
+        'test_worker1' => '3',
+        'test_worker2' => '3',
+        'test_worker3' => '3',
+        'test_worker4' => '3',
+        'test_worker5' => '3',
+        'test_worker6' => '3'
+        }
+      }
+    }
+
+    let(:test_client) {
+      Redis.new
+    }
+
+    let(:adapter1) {
+      adapter1 = Gru::Adapters::RedisAdapter.new(Gru::Configuration.new(settings.clone))
+      allow(adapter1).to receive(:hostname).and_return('test1')
+      adapter1
+    }
+
+    let(:adapter2) {
+      adapter2 = Gru::Adapters::RedisAdapter.new(Gru::Configuration.new(settings.clone))
+      allow(adapter2).to receive(:hostname).and_return('test2')
+      adapter2
+    }
+
+    let(:adapter3) {
+      adapter3 = Gru::Adapters::RedisAdapter.new(Gru::Configuration.new(settings.clone))
+      allow(adapter3).to receive(:hostname).and_return('test3')
+      adapter3
+    }
+
+    let(:adapter4) {
+      adapter4 = Gru::Adapters::RedisAdapter.new(Gru::Configuration.new(settings.clone))
+      allow(adapter4).to receive(:hostname).and_return('test4')
+      adapter4
+    }
+
+    it "honors the max workers per host setting" do
+      test1 = Gru::WorkerManager.new(adapter1)
+      test2 = Gru::WorkerManager.new(adapter2)
+      test3 = Gru::WorkerManager.new(adapter3)
+      test4 = Gru::WorkerManager.new(adapter4)
+      test1.register_workers
+      test2.register_workers
+      test3.register_workers
+      # Honors max processes per host setting
+      expect(test1.adjust_workers).to eq({
+        'test_worker1' => 3,
+        'test_worker2' => 2,
+        'test_worker3' => 0,
+        'test_worker4' => 0,
+        'test_worker5' => 0,
+        'test_worker6' => 0
+      })
+
+      expect(test2.adjust_workers).to eq({
+        'test_worker1' => 0,
+        'test_worker2' => 1,
+        'test_worker3' => 3,
+        'test_worker4' => 1,
+        'test_worker5' => 0,
+        'test_worker6' => 0
+      })
+
+      expect(test3.adjust_workers).to eq({
+        'test_worker1' => 0,
+        'test_worker2' => 0,
+        'test_worker3' => 0,
+        'test_worker4' => 2,
+        'test_worker5' => 3,
+        'test_worker6' => 0
+      })
+
+      test4.register_workers
+      expect(test4.adjust_workers).to eq({
+        'test_worker1' => 0,
+        'test_worker2' => 0,
+        'test_worker3' => 0,
+        'test_worker4' => 0,
+        'test_worker5' => 0,
+        'test_worker6' => 3
+      })
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gru
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Jeffrey Gillis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-21 00:00:00.000000000 Z
+date: 2016-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis