gearman-ruby 4.0.4 → 4.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f8799efeb8ec1766ffb262f401595cf9f1029fdc
-  data.tar.gz: e672b3358f6bda8dc55467e1a5af72973d422e2d
+  metadata.gz: 6ffdbe675e2808d933b8fb7ebdd8ffe5a42b0814
+  data.tar.gz: a1633bb7cb2c83ecc631e8f32bb15719d99a06c2
 SHA512:
-  metadata.gz: efea3f3ae2c88664d13b56debf6ec69e533c43136c8b2555f6c93de7ade16406e51ccee0fd27d3a40cecb07f347281f92139b8b30a2cc6a12c1f9ac49c1d36a8
-  data.tar.gz: 5e391604849cec1c2ae53b300d9c0812b2a9f55b9a76faa0f9843b48cc4368a82f44adc0d545ad9adbdbae54552ae6115089365361a725c1180f985193a0b66a
+  metadata.gz: d7ef5d38b28304b96945bc9a7d9c023acec174ff6dbc3585c19dbb123d2dd2e6aeeb70edfc7e0ed5e81d2bfe7f65b424d8ddac75b5e8993864808cc1f13f3dbd
+  data.tar.gz: bed878e4570169263cf3106ad7db091204d7533db1bdff81be6c882f3a8a96b9833ea679b2807423a5218201d4a80efe9e50adc5b43effdf981d04076826a078

data/examples/client_dedupe.rb

@@ -0,0 +1,22 @@
+$LOAD_PATH.unshift("../lib")
+require 'rubygems'
+require 'gearman'
+
+# Control logger
+l = Logger.new($stdout)
+l.level = Logger::DEBUG
+Gearman.logger=l
+
+servers = ['localhost:4730']
+  
+client = Gearman::Client.new(servers)
+
+# Submitting work with no explicit unique ID will
+# cause one to be generated by the client based on
+# the job queue and the data, these 10 jobs should
+# get de-duped into one job in the server (provided
+# that nobody is pulling from the queue in the middle)
+(0..10).each do |i|
+  task = Gearman::BackgroundTask.new('dedupe', 'data')
+  client.do_task(task)
+end

data/examples/worker_dedupe.rb

@@ -0,0 +1,19 @@
+$LOAD_PATH.unshift("../lib")
+require 'rubygems'
+require 'gearman'
+
+# Control logger
+l = Logger.new($stdout)
+l.level = Logger::DEBUG
+Gearman.logger=l
+
+servers = ['localhost:4730']
+  
+worker = Gearman::Worker.new(servers)
+
+worker.add_ability('dedupe') do |data, job|
+  puts "Should only see one of these come through!"
+  true
+end
+
+loop { worker.work }

data/lib/gearman/client.rb

@@ -98,7 +98,7 @@ module Gearman
           raise ProtocolError, message
         end
       rescue NetworkError
-        message = "Network error on read from #{hostport} while adding job, marking server bad"
+        message = "Network error on read from #{connection.hostport} while adding job, marking server bad"
         logger.error message
         raise NetworkError, message
       rescue NoJobServersError

data/lib/gearman/connection_pool.rb

@@ -5,6 +5,8 @@ module Gearman
     include Logging
 
     DEFAULT_PORT = 4730
+    TIME_BETWEEN_CHECKS = 60 #seconds
+    SLEEP_TIME = 30 #seconds
 
     def initialize(servers = [])
       @bad_servers          = []
@@ -14,9 +16,9 @@ module Gearman
       @reconnect_seconds    = 10
       @server_counter       = 0   # Round-robin distribution of requests
       @servers_mutex        = Mutex.new
+      @last_check_time      = Time.now
 
       add_servers(servers)
-      start_reconnect_thread
     end
 
     def add_connection(connection)
@@ -49,7 +51,6 @@ module Gearman
 
     def get_connection(coalesce_key = nil)
       @servers_mutex.synchronize do
-
         logger.debug "Available job servers: #{@job_servers.inspect}"
         raise NoJobServersError if @job_servers.empty?
         @server_counter += 1
@@ -62,6 +63,7 @@ module Gearman
     end
 
     def poll_connections(timeout = nil)
+      update_job_servers
       available_sockets = []
       @servers_mutex.synchronize do
         available_sockets.concat @job_servers.collect { |conn| conn.socket }
@@ -73,6 +75,7 @@ module Gearman
     end
 
     def with_all_connections(&block)
+      update_job_servers
       @servers_mutex.synchronize do
         @job_servers.each do |connection|
           begin
@@ -88,6 +91,10 @@ module Gearman
 
     private
 
+      def time_to_check_connections
+        (Time.now - @last_check_time) >= TIME_BETWEEN_CHECKS
+      end
+
       def deactivate_connection(connection)
         @job_servers.reject! { |c| c == connection }
         @bad_servers << connection
@@ -99,31 +106,35 @@ module Gearman
         @connection_handler.call(connection) if @connection_handler
       end
 
-      def start_reconnect_thread
-        Thread.new do
-          loop do
-            @servers_mutex.synchronize do
-              # If there are any failed servers, try to reconnect to them.
-              update_job_servers unless @bad_servers.empty?
-            end
-            sleep @reconnect_seconds
-          end
-        end.run
-      end
-
+      ##
+      # Check for bad servers and update the available pools
+      #
       def update_job_servers
+        # Check if it's been > TIME_BETWEEN_CHECKS or we have no good servers
+        return unless time_to_check_connections || @job_servers.empty?
+
         logger.debug "Found #{@bad_servers.size} zombie connections, checking pulse."
-        @bad_servers.each do |connection|
-          begin
-            message = "Testing server #{connection}..."
-            if connection.is_healthy?
-              logger.debug "#{message} Connection is healthy, putting back into service"
-              activate_connection(connection)
-            else
-              logger.debug "#{message} Still down."
+        @servers_mutex.synchronize do
+          @bad_servers.each do |connection|
+            begin
+              message = "Testing server #{connection}..."
+              if connection.is_healthy?
+                logger.debug "#{message} Connection is healthy, putting back into service"
+                activate_connection(connection)
+              else
+                logger.debug "#{message} Still down."
+              end
             end
           end
         end
+
+        # Sleep for a few to allow a chance for the world to become sane
+        if @job_servers.empty?
+          logger.warn "No job servers available, sleeping for #{SLEEP_TIME} seconds"
+          sleep(SLEEP_TIME)
+        end
+
+        @last_check_time = Time.now
       end
 
 

data/lib/gearman/version.rb

@@ -1,3 +1,3 @@
 module Gearman
-  VERSION = '4.0.4'
+  VERSION = '4.0.5'
 end

data/spec/client_spec.rb

@@ -51,6 +51,19 @@ describe Gearman::Client do
 
   end
 
+  it 'propagates NetworkErrors while submitting jobs' do
+    mock_connection = double(Gearman::Connection)
+    mock_connection.stub(:send_request).and_raise(Gearman::NetworkError)
+    mock_connection.stub(:hostport).and_return("localhost:4730")
+
+    @mock_connection_pool.stub(:get_connection) { mock_connection }
+
+    task = Gearman::Task.new("queue", "data")
+
+    expect {
+      @client.submit_job(task)
+    }.to raise_error(Gearman::NetworkError)
+  end
 
 
   it "should raise a NetworkError when it didn't write as much as expected to a socket" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gearman-ruby
 version: !ruby/object:Gem::Version
-  version: 4.0.4
+  version: 4.0.5
 platform: ruby
 authors:
 - John Ewart
@@ -27,8 +27,10 @@ files:
 - Rakefile
 - examples/client.rb
 - examples/client_background_jobs.rb
+- examples/client_dedupe.rb
 - examples/client_reverse_string.rb
 - examples/worker.rb
+- examples/worker_dedupe.rb
 - examples/worker_reverse_string.rb
 - gearman-ruby.gemspec
 - lib/gearman.rb