gru 0.0.10 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 13495796cf42eb14f84201efe3ff4701a3a4b2e5
-  data.tar.gz: d965fb4d6e6003907e3b4253a6741c0d93549b1e
+  metadata.gz: 3568199a19c30ad45c3e4888884901d19ebf8c2c
+  data.tar.gz: 987f4860c98601be7dca13be6dcd4ec590f84881
 SHA512:
-  metadata.gz: faa6fec2b08740761f6a436ad5fba23586cd4ead30a3c7f45814b66180e2cfc4de738a834c2a1eab51b5710fa79c701b146937448c759f8f5a7ee871fccfa9e9
-  data.tar.gz: b369f35ebec7200cfae1ff6b02fb7f079a05f440648b575245bf8eb6f7a0982a06668b44872c466c0d5d5312d8976547234a7869a0ccb4685ce4b184044b2f96
+  metadata.gz: 386d66bc3221a042fc314ff088dfec031a68def5245873dfb3ba99af9a8567a49096ac9e6f8d97ef038ea077ee8ac429c5e62b09803afb2a9d270b5436940238
+  data.tar.gz: a2ffe6817185cd9c44fd3aaa0968b17bf49a6e27ba004dcaca2007d5998f4f8a6aeffe9e3fa03b73f601ba7d60f6be2e8f7a773a9783a236d36778342e2fe038

data/lib/gru/adapters/redis_adapter.rb

@@ -5,9 +5,9 @@ module Gru
     class RedisAdapter
       attr_reader :client
 
-      def initialize(client,settings)
-        @client = client
+      def initialize(settings)
         @settings = settings
+        @client = initialize_client(settings.client_settings)
       end
 
       def set_worker_counts
@@ -18,6 +18,7 @@ module Gru
         set_max_worker_counts(@settings.host_maximums)
         register_global_workers(@settings.cluster_maximums)
         set_max_global_worker_counts(@settings.cluster_maximums)
+        update_heartbeat if manage_heartbeat?
       end
 
       def provision_workers
@@ -51,32 +52,31 @@ module Gru
       def release_workers
         workers = max_host_workers
         workers.keys.each do |worker|
-          host_running_count = local_running_count(worker)
-          running_count = host_running_count
-          global_running_count = host_running_count
-          host_running_count.times do
-            if global_running_count > 0
-              global_running_count = send_message(:hincrby, global_workers_running_key,worker,-1)
-            end
-            if running_count > 0
-              running_count = send_message(:hincrby, host_workers_running_key,worker,-1)
-            end
+          host_count = local_running_count(worker)
+          global_count = host_count
+          host_count.times do
+            global_count = send_message(:hincrby, global_workers_running_key, worker, -1) if global_count > 0
+            host_count = send_message(:hincrby, host_workers_running_key,worker,-1) if host_count > 0
           end
         end
         send_message(:del, host_workers_running_key)
         send_message(:del, host_max_worker_key)
+        send_message(:hdel, heartbeat_key, hostname)
       end
 
-      def release_presumed_dead_workers
-        presumed_dead_cluster_members.each_pair do |hostname,timestamp|
+      def release_presumed_dead_worker_hosts
+        return false unless manage_heartbeat?
+        update_heartbeat
+        presumed_dead_worker_hosts.each_pair do |hostname,timestamp|
           lock_key = "#{gru_key}:removing_dead_host:#{hostname}"
           if send_message(:setnx,lock_key,Time.now.to_i)
-            remove_dead_host_workers_from_counts(hostname)
+            remove_worker_host(hostname)
+            send_message(:hdel,heartbeat_key,hostname)
             send_message(:del,lock_key)
             return true
           end
-          false
         end
+        false
       end
 
       private
@@ -122,17 +122,24 @@ module Gru
         send_message(:hset,global_max_worker_key,worker,count)
       end
 
-      def remove_dead_host_workers_from_counts(hostname)
-        lock_key = "#{gru_key}:#{hostname}"
-        if send_message(:setnx,lock_key,Time.now.to_i)
-          workers_running_on_dead_host = send_message(:hgetall, "#{gru_key}:#{hostname}:workers_running")
-          workers_running_on_dead_host.each_pair do |worker_name, count|
-            send_message(:hincrby,"#{gru_key}:#{hostname}:workers_running",worker_name,Integer(count)*-1)
-            send_message(:hincrby,global_workers_running_key,worker_name,Integer(count)*-1)
+      def manage_heartbeat?
+        @settings.manage_worker_heartbeats
+      end
+
+      def update_heartbeat
+        send_message(:hset,heartbeat_key,hostname,Time.now.to_i)
+      end
+
+      def remove_worker_host(hostname)
+        workers = send_message(:hgetall, "#{gru_key}:#{hostname}:workers_running")
+        workers.each_pair do |worker_name, count|
+          local_count, global_count = Integer(count), Integer(count)
+          Integer(count).times do
+            local_count = send_message(:hincrby,"#{gru_key}:#{hostname}:workers_running",worker_name,-1) if local_count > 0
+            global_count = send_message(:hincrby,global_workers_running_key,worker_name,-1) if global_count > 0
           end
-          send_message(:del,lock_key)
-          send_message(:hdel,resque_cluster_pings_key,hostname)
         end
+        send_message(:del,"#{gru_key}:#{hostname}:workers_running")
       end
 
       def reset_removed_global_worker_counts(workers)
@@ -175,8 +182,8 @@ module Gru
         send_message(:hgetall,global_max_worker_key)
       end
 
-      def resque_cluster_members
-        send_message(:hgetall, resque_cluster_pings_key)
+      def workers_with_heartbeats
+        send_message(:hgetall, heartbeat_key)
       end
 
       def reserve_worker?(worker)
@@ -215,8 +222,8 @@ module Gru
         counts
       end
 
-      def presumed_dead_cluster_members
-        resque_cluster_members.select{ |hostname, timestamp| Time.parse(timestamp).to_i + presume_host_dead_after < Time.now.to_i}
+      def presumed_dead_worker_hosts
+        workers_with_heartbeats.select{ |hostname, timestamp| timestamp.to_i + presume_host_dead_after < Time.now.to_i}
       end
 
       def local_running_count(worker)
@@ -270,12 +277,12 @@ module Gru
         "#{gru_key}:#{hostname}"
       end
 
-      def gru_key
-        "GRU:#{@settings.environment_name}:#{@settings.cluster_name}"
+      def heartbeat_key
+        "#{gru_key}:heartbeats"
       end
 
-      def resque_cluster_pings_key
-        "resque:cluster:#{@settings.cluster_name}:#{@settings.environment_name}:pings"
+      def gru_key
+        "GRU:#{@settings.environment_name}:#{@settings.cluster_name}"
       end
 
       def hostname
@@ -286,6 +293,9 @@ module Gru
         @client.send(action,*args)
       end
 
+      def initialize_client(config=nil)
+        Redis.new(config || {})
+      end
     end
   end
 end

data/lib/gru/configuration.rb

@@ -3,25 +3,20 @@ require 'gru/adapters/redis_adapter'
 
 module Gru
   class Configuration
-    attr_reader :cluster_maximums, :host_maximums, :rebalance_flag, :adapter, :cluster_name, :environment_name, :presume_host_dead_after
+    attr_reader :cluster_maximums, :host_maximums, :rebalance_flag, :adapter, :cluster_name, :environment_name, :presume_host_dead_after, :client_settings, :manage_worker_heartbeats
     def initialize(settings)
       @host_maximums = settings.delete(:host_maximums) || settings.delete(:cluster_maximums)
       @cluster_maximums = settings.delete(:cluster_maximums) || @host_maximums
       @rebalance_flag = settings.delete(:rebalance_flag) || false
       @cluster_name = settings.delete(:cluster_name) || 'default'
       @environment_name = settings.delete(:environment_name) || 'default'
-      client = initialize_client(settings.delete(:client_settings))
       @presume_host_dead_after = settings.delete(:presume_host_dead_after)
-      @adapter = Gru::Adapters::RedisAdapter.new(client,self)
+      @client_settings = settings.delete(:client_settings)
+      @manage_worker_heartbeats = settings.delete(:manage_worker_heartbeats) || false
+      @adapter = Gru::Adapters::RedisAdapter.new(self)
       if @cluster_maximums.nil?
-        raise ArgumentError "Need at least a cluster configuration"
+        raise ArgumentError, "Need at least a cluster configuration"
       end
     end
-
-    private
-
-    def initialize_client(config=nil)
-      Redis.new(config || {})
-    end
   end
 end

data/lib/gru/version.rb

@@ -1,3 +1,3 @@
 module Gru
-  VERSION = "0.0.10"
+  VERSION = "0.1.0"
 end

data/lib/gru/worker_manager.rb

@@ -7,7 +7,7 @@ module Gru
     end
 
     def expire_dead_cluster_members
-      @adapter.release_presumed_dead_workers
+      @adapter.release_presumed_dead_worker_hosts
     end
 
     def register_workers

data/spec/gru/adapters/redis_adapter_spec.rb

@@ -5,7 +5,7 @@ require 'pry'
 describe Gru::Adapters::RedisAdapter do
   before(:each) do
     allow(Socket).to receive(:gethostname).and_return(hostname)
-    allow_any_instance_of(Gru::Configuration).to receive(:initialize_client).and_return(client)
+    allow_any_instance_of(Gru::Adapters::RedisAdapter).to receive(:initialize_client).and_return(client)
   end
 
   let(:hostname) { 'foo' }
@@ -19,14 +19,13 @@ describe Gru::Adapters::RedisAdapter do
   }
 
   let(:adapter) {
-    Gru::Adapters::RedisAdapter.new(client,config)
+    Gru::Adapters::RedisAdapter.new(config)
   }
 
   let(:gru_key) {
     "GRU:#{config.environment_name}:#{config.cluster_name}"
   }
 
-
   context "initialization" do
     it "has a client" do
       expect(adapter.client).to eq(client)
@@ -227,7 +226,6 @@ describe Gru::Adapters::RedisAdapter do
       expect(client).to receive(:del).with("#{gru_key}:test_worker").exactly(3).times
       expect(client).to receive(:get).with("#{gru_key}:rebalance").and_return("true").exactly(3).times
       adapter.provision_workers
-
     end
 
     it "increases load when workers are removed" do

data/spec/gru/gru_integration_spec.rb

@@ -0,0 +1,171 @@
+require './lib/gru.rb'
+require 'socket'
+require 'digest'
+require 'redis'
+require 'pry'
+
+
+# This requires a redis server instance running on localhost
+# un-pend to run with actual redis-server
+xdescribe Gru do
+
+  after {
+    client.flushdb
+  }
+
+  let(:settings) {
+    {
+      cluster_maximums: {
+      'test_worker' => '3'
+      }
+    }
+  }
+
+  let(:client) {
+    Redis.new
+  }
+
+  let(:hostname) {
+    Socket.gethostname
+  }
+
+  let(:host_key) {
+    "GRU:default:default:#{hostname}"
+  }
+
+  let(:global_key) {
+    "GRU:default:default:global"
+  }
+
+  context "configuration" do
+    it "sets configuration in redis" do
+      manager = Gru.create(settings.clone)
+      expect(client.hgetall("#{host_key}:max_workers")).to eq(settings[:cluster_maximums])
+      expect(manager.provision_workers).to eq({ 'test_worker' => 3 })
+      expect(client.hgetall("#{host_key}:workers_running")).to eq(settings[:cluster_maximums])
+    end
+  end
+
+  context "adjusting worker counts" do
+    it "adjusts worker counts based on changing global counts" do
+      manager = Gru.create(settings.clone)
+      expect(manager.provision_workers).to eq({ 'test_worker' => 3 })
+      client.hset("#{global_key}:max_workers", 'test_worker', 1)
+      expect(manager.adjust_workers).to eq({ 'test_worker' => -2})
+      expect(manager.adjust_workers).to eq({ 'test_worker' => 0})
+      expect(client.hget("#{global_key}:workers_running", 'test_worker')).to eq("1")
+      client.hset("#{global_key}:max_workers", 'test_worker', 3)
+      expect(manager.adjust_workers).to eq({ 'test_worker' => 2})
+      expect(client.hget("#{global_key}:workers_running", 'test_worker')).to eq("3")
+    end
+
+    it "adjusts worker counts based on changing local counts" do
+      manager = Gru.create(settings.clone)
+      expect(manager.provision_workers).to eq({ 'test_worker' => 3 })
+      client.hset("#{host_key}:max_workers", 'test_worker', 1)
+      expect(manager.adjust_workers).to eq({ 'test_worker' => -1})
+      expect(manager.adjust_workers).to eq({ 'test_worker' => -1})
+      expect(manager.adjust_workers).to eq({ 'test_worker' => 0})
+      expect(client.hget("#{global_key}:workers_running", 'test_worker')).to eq("1")
+      expect(client.hget("#{host_key}:workers_running", 'test_worker')).to eq("1")
+    end
+
+    it "does not exceed local maximum counts" do
+      manager = Gru.create(settings.clone)
+      expect(manager.provision_workers).to eq({ 'test_worker' => 3 })
+      client.hset("#{host_key}:max_workers", 'test_worker', 2)
+      expect(manager.adjust_workers).to eq({ 'test_worker' => -1})
+      expect(manager.adjust_workers).to eq({ 'test_worker' => 0})
+      client.hset("#{global_key}:max_workers", 'test_worker', 4)
+      expect(manager.adjust_workers).to eq({ 'test_worker' => 0})
+      expect(client.hget("#{global_key}:workers_running", 'test_worker')).to eq("2")
+      expect(client.hget("#{host_key}:workers_running", 'test_worker')).to eq("2")
+    end
+
+    it "does not exceed global maximum counts" do
+      manager = Gru.create(settings.clone)
+      expect(manager.provision_workers).to eq({ 'test_worker' => 3 })
+      client.hset("#{host_key}:max_workers", 'test_worker', 0)
+      expect(manager.adjust_workers).to eq({ 'test_worker' => 0})
+      client.hset("#{global_key}:max_workers", 'test_worker', 3)
+      expect(manager.adjust_workers).to eq({ 'test_worker' => 0})
+      client.hset("#{host_key}:max_workers", 'test_worker', 4)
+      expect(manager.adjust_workers).to eq({ 'test_worker' => 0})
+      expect(client.hget("#{global_key}:workers_running", 'test_worker')).to eq("3")
+      expect(client.hget("#{host_key}:workers_running", 'test_worker')).to eq("3")
+    end
+  end
+
+  context "multiple workers" do
+    let(:settings) {
+      {
+        rebalance_flag: true,
+        manage_worker_heartbeats: true,
+        cluster_maximums: {
+        'test_worker' => '6'
+        }
+      }
+    }
+
+    let(:test_client) {
+      Redis.new
+    }
+
+    let(:adapter1) {
+      adapter1 = Gru::Adapters::RedisAdapter.new(Gru::Configuration.new(settings.clone))
+      allow(adapter1).to receive(:hostname).and_return('test1')
+      adapter1
+    }
+
+    let(:adapter2) {
+      adapter2 = Gru::Adapters::RedisAdapter.new(Gru::Configuration.new(settings.clone))
+      allow(adapter2).to receive(:hostname).and_return('test2')
+      adapter2
+    }
+
+    let(:adapter3) {
+      adapter3 = Gru::Adapters::RedisAdapter.new(Gru::Configuration.new(settings.clone))
+      allow(adapter3).to receive(:hostname).and_return('test3')
+      adapter3
+    }
+
+    it "adjusts workers when new hosts are added" do
+      test1 = Gru::WorkerManager.new(adapter1)
+      test2 = Gru::WorkerManager.new(adapter2)
+      test3 = Gru::WorkerManager.new(adapter3)
+      test1.register_workers
+      test2.register_workers
+      expect(test_client.hget('GRU:default:default:test1:workers_running', 'test_worker')).to eq('0')
+      expect(test1.adjust_workers).to eq( { 'test_worker' => 3 })
+      expect(test2.adjust_workers).to eq( { 'test_worker' => 3 })
+      test3.register_workers
+      expect(test1.adjust_workers).to eq( { 'test_worker' => -1 })
+      expect(test1.adjust_workers).to eq( { 'test_worker' => 0 })
+      expect(test3.adjust_workers).to eq( { 'test_worker' => 1 })
+      expect(test3.adjust_workers).to eq( { 'test_worker' => 0 })
+      expect(test2.adjust_workers).to eq( { 'test_worker' => -1 })
+      expect(test2.adjust_workers).to eq( { 'test_worker' => 0 })
+      expect(test3.adjust_workers).to eq( { 'test_worker' => 1 })
+      expect(test3.adjust_workers).to eq( { 'test_worker' => 0 })
+      expect(test1.adjust_workers).to eq( { 'test_worker' => 0 })
+      expect(test2.adjust_workers).to eq( { 'test_worker' => 0 })
+    end
+
+    it "recovers lost worker counts" do
+      test1 = Gru::WorkerManager.new(adapter1)
+      test2 = Gru::WorkerManager.new(adapter2)
+      test3 = Gru::WorkerManager.new(adapter3)
+      test1.register_workers
+      test2.register_workers
+      expect(test1.adjust_workers).to eq( { 'test_worker' => 3 })
+      expect(test2.adjust_workers).to eq( { 'test_worker' => 3 })
+      test_client.hset("GRU:default:default:heartbeats", 'test1', Time.now - 300)
+      expect(test2.adjust_workers).to eq( { 'test_worker' => 3 })
+      test3.register_workers
+      expect(test3.adjust_workers).to eq( { 'test_worker' => 0 })
+      expect(test2.adjust_workers).to eq( { 'test_worker' => -3 })
+      test_client.hset("GRU:default:default:heartbeats", 'test3', Time.now - 300)
+      expect(test2.adjust_workers).to eq( { 'test_worker' => 3 })
+    end
+  end
+end

data/spec/gru/worker_manager_spec.rb

@@ -31,7 +31,7 @@ describe Gru::WorkerManager do
     it "determines new workers to create" do
       expect(adapter).to receive(:provision_workers).and_return({})
       expect(adapter).to receive(:expire_workers).and_return({})
-      expect(adapter).to receive(:release_presumed_dead_workers)
+      expect(adapter).to receive(:release_presumed_dead_worker_hosts)
       manager.adjust_workers
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gru
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.1.0
 platform: ruby
 authors:
 - Jeffrey Gillis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-18 00:00:00.000000000 Z
+date: 2016-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -114,6 +114,7 @@ files:
 - lib/gru/version.rb
 - lib/gru/worker_manager.rb
 - spec/gru/adapters/redis_adapter_spec.rb
+- spec/gru/gru_integration_spec.rb
 - spec/gru/worker_manager_spec.rb
 homepage: ''
 licenses:
@@ -141,4 +142,5 @@ specification_version: 4
 summary: An atomic worker/minion manager.
 test_files:
 - spec/gru/adapters/redis_adapter_spec.rb
+- spec/gru/gru_integration_spec.rb
 - spec/gru/worker_manager_spec.rb