gru 0.0.5 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2002057fd3f7622545619c577dd9116f5efafea3
-  data.tar.gz: c1eec4430b446adbf7f92bd3f009ef3b9f2722c6
+  metadata.gz: 4e79626380c84c31c2f12ca59d88a1b36dd3294f
+  data.tar.gz: 9e5903acfec6906910e8fd6050dd2d83d40e60de
 SHA512:
-  metadata.gz: e3f077b9c3f83e60a08f733a094d8e1382025dbce996ee1808222fccd43131a892ccbb36298c4828e78ce29d1055f10480f13fa8213f333fcd21e6223e782e5b
-  data.tar.gz: e090daf35b63bdab1c4920008125b9458958d73160026b833e8484e50d4cf4753075c73455e363987d796b64a3bc63a84543904cc86be8dc333ec430ab9ede7d
+  metadata.gz: 65c48ef82fe013ba1ef8735c41cfe9834a73a824c3aa67c3ca4f5ab4ca9d0e1ac9d25b54199c032d46083f97698be3dc8e53dfd32e64c90e9170b0068b5934ff
+  data.tar.gz: 2d4d11082bf6c6ac06b6e53ca5dbbeaa7d1ddc029553a6aac7335b7a9856a35e5051bb1f3028a6b8090464888fe670759cc2c732575d5a66da96af0deacd34a6

data/lib/gru/adapters/redis_adapter.rb

@@ -12,6 +12,7 @@ module Gru
 
       def set_worker_counts
         set_rebalance_flag(@settings.rebalance_flag)
+        set_presume_host_dead_after(@settings.presume_host_dead_after)
         release_workers
         register_workers(@settings.host_maximums)
         set_max_worker_counts(@settings.host_maximums)
@@ -66,6 +67,18 @@ module Gru
         send_message(:del, host_max_worker_key)
       end
 
+      def release_presumed_dead_workers
+        presumed_dead_cluster_members.each_pair do |hostname,timestamp|
+          lock_key = "#{gru_key}:removing_dead_host:#{hostname}"
+          if send_message(:setnx,lock_key,Time.now.to_i)
+            remove_dead_host_workers_from_counts(hostname)
+            send_message(:del,lock_key)
+            return true
+          end
+          false
+        end
+      end
+
       private
 
       def register_workers(workers)
@@ -89,6 +102,10 @@ module Gru
         send_message(:set,"#{gru_key}:rebalance",rebalance)
       end
 
+      def set_presume_host_dead_after(seconds)
+        send_message(:set,"#{gru_key}:presume_host_dead_after",seconds)
+      end
+
       def register_worker(worker,count)
         send_message(:hsetnx,host_workers_running_key,worker,count)
       end
@@ -105,6 +122,15 @@ module Gru
         send_message(:hset,global_max_worker_key,worker,count)
       end
 
+      def remove_dead_host_workers_from_counts(hostname)
+        workers_running_on_dead_host = send_message(:hgetall, "#{gru_key}:#{hostname}:workers_running")
+        workers_running_on_dead_host.each_pair do |worker_name, count|
+          send_message(:hincrby,"#{gru_key}:#{hostname}:workers_running",worker_name,Integer(count)*-1)
+          send_message(:hincrby,global_workers_running_key,worker_name,Integer(count)*-1)
+        end
+        send_message(:hdel,resque_cluster_pings_key,hostname)
+      end
+
       def reset_removed_global_worker_counts(workers)
         global_max = max_host_workers
         global_max.each_pair do |worker, count|
@@ -145,6 +171,10 @@ module Gru
         send_message(:hgetall,global_max_worker_key)
       end
 
+      def resque_cluster_members
+        send_message(:hgetall, resque_cluster_pings_key)
+      end
+
       def reserve_worker?(worker)
         host_running,global_running,host_max,global_max = worker_counts(worker)
         result = false
@@ -176,6 +206,10 @@ module Gru
         end
       end
 
+      def presumed_dead_cluster_members
+        resque_cluster_members.select{ |hostname, timestamp| Time.parse(timestamp).to_i + presume_host_dead_after < Time.now.to_i}
+      end
+
       def local_running_count(worker)
         send_message(:hget,host_workers_running_key,worker).to_i
       end
@@ -194,6 +228,11 @@ module Gru
         send_message(:get,"#{gru_key}:rebalance") == "true"
       end
 
+      def presume_host_dead_after
+        dead_after_number_of_seconds = send_message(:get,"#{gru_key}:presume_host_dead_after").to_i
+        dead_after_number_of_seconds > 0 ? dead_after_number_of_seconds : 120
+      end
+
       def host_max_worker_key
         "#{host_key}:max_workers"
       end
@@ -222,6 +261,10 @@ module Gru
         "GRU:#{@settings.environment_name}:#{@settings.cluster_name}"
       end
 
+      def resque_cluster_pings_key
+        "resque:cluster:#{@settings.cluster_name}:#{@settings.environment_name}:pings"
+      end
+
       def hostname
         @hostname ||= Socket.gethostname
       end

data/lib/gru/configuration.rb

@@ -3,7 +3,7 @@ require 'gru/adapters/redis_adapter'
 
 module Gru
   class Configuration
-    attr_reader :cluster_maximums, :host_maximums, :rebalance_flag, :adapter, :cluster_name, :environment_name
+    attr_reader :cluster_maximums, :host_maximums, :rebalance_flag, :adapter, :cluster_name, :environment_name, :presume_host_dead_after
     def initialize(settings)
       @host_maximums = settings.delete(:host_maximums) || settings.delete(:cluster_maximums)
       @cluster_maximums = settings.delete(:cluster_maximums) || @host_maximums
@@ -11,6 +11,7 @@ module Gru
       @cluster_name = settings.delete(:cluster_name) || 'default'
       @environment_name = settings.delete(:environment_name) || 'default'
       client = initialize_client(settings.delete(:client_settings))
+      @presume_host_dead_after = settings.delete(:presume_host_dead_after)
       @adapter = Gru::Adapters::RedisAdapter.new(client,self)
       if @cluster_maximums.nil?
         raise ArgumentError "Need at least a cluster configuration"

data/lib/gru/version.rb

@@ -1,3 +1,3 @@
 module Gru
-  VERSION = "0.0.5"
+  VERSION = "0.0.7"
 end

data/lib/gru/worker_manager.rb

@@ -6,6 +6,10 @@ module Gru
       @adapter = adapter
     end
 
+    def expire_dead_cluster_members
+      @adapter.release_presumed_dead_workers
+    end
+
     def register_workers
       @adapter.set_worker_counts
     end
@@ -20,6 +24,7 @@ module Gru
 
     def adjust_workers
       result = {}
+      expire_dead_cluster_members
       add = provision_workers
       remove = expire_workers
       keys = add.keys + remove.keys

metadata

@@ -1,97 +1,97 @@
 --- !ruby/object:Gem::Specification
 name: gru
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.7
 platform: ruby
 authors:
 - Jeffrey Gillis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-25 00:00:00.000000000 Z
+date: 2015-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - '>'
       - !ruby/object:Gem::Version
         version: '0.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - '>'
       - !ruby/object:Gem::Version
         version: '0.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.7'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 3.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 3.1.0
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - '>'
       - !ruby/object:Gem::Version
         version: '0.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - '>'
       - !ruby/object:Gem::Version
         version: '0.0'
 - !ruby/object:Gem::Dependency
   name: awesome_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - '>'
       - !ruby/object:Gem::Version
         version: '0.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - '>'
       - !ruby/object:Gem::Version
         version: '0.0'
 description: This is a worker/minion manager using different atomic data stores.
@@ -101,7 +101,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
+- .gitignore
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -125,20 +125,21 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.4
 signing_key: 
 specification_version: 4
 summary: An atomic worker/minion manager.
 test_files:
 - spec/gru/adapters/redis_adapter_spec.rb
 - spec/gru/worker_manager_spec.rb
+has_rdoc: