gearman-ruby 3.0.8 → 4.0.2

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MzdjNjBlZDA5ODQ0ZGZhM2UwMzk0M2ZlNTNlNDY5NmEwNmMzODI4NA==
+    ZWIzNDljYTkyOTdjY2ZlMmJmYzMzNTIwMTlmOTdkMmQwZGE1MGViNw==
   data.tar.gz: !binary |-
-    YmQ2ZjZiOTUxMTI5ZWRmYjRjZTE1MTY5ZGFhMGYwMWRjNGVjYzJmZA==
+    ODc1MGU0ZjM2OWQ4ZmYxZjlmYTNkOTNhZjk1OGEyZDlkNDRhYjYzZQ==
 SHA512:
   metadata.gz: !binary |-
-    OGYwY2QxNDc2MmMxYzEyODRkNzU2NWMyYmY5MTNlYTY2ODRmY2ExNGZmMDk0
-    YmEzOTlhZTk0YzIzZjAxMTQ1YWZjMjUyNGYzZWZmYWFlZTM1ZDI5NDliMjUz
-    ZmM0MDUzZGJhNzkzZTQxZjg2YjJjNDY4NmJjNmM5MDc1MDM0NDU=
+    YThkMjM1NGY4NGRlM2M4MGE0ODg5YzRiZDJlYWZkNzU2YmViNDlkZjhmZDk3
+    NjQ5MWE4YmNmOTJlNzM2OGI5ODI2NzBjNTAxZTRhNGY1MWU5ZWM4NTA4MDQy
+    MjI2MTk3YzNkZWNlNzkzZjFlMjAzOWIxZmZiNGY4MDYwNGI3ZTE=
   data.tar.gz: !binary |-
-    NzUzMDE1YmUyYTE4MDcxOWRlYjU5ZGQ2YzBiZTdlNTFiNTVjOTY0YmExZTdk
-    ZGQ2YzBmMzQ2YzlkMjZmZDU4NmZlMGUxOGJhZjM1MGJkYTJmZWEzNDJkZmUy
-    Y2E0N2ZlNjY3MTMxYzk1OWM5MDQxZTZjYmEzYTUwYTU4ODhkZWQ=
+    MmE3MmU0OTVkNDljMGEwODg2YWQ4OGQ1MjI3YWVhZWEyMDZlZjliYzIzYTk2
+    ZGUxNjkzZjgwNDRlMWY5OTAwMTM5NzI0NThiYmFhM2NkOWQ3YmUzODhhYzY2
+    NWZkODg5YjgzMzEyMDNhY2QwYzAxZTM3MmNhMTk1OWNkODMzOTM=

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+ - 2.1.0
+ - 2.0.0
+ - 1.9.3

data/CHANGELOG.md

@@ -1,5 +1,7 @@
 # CHANGES 
-#
-## 3.0.8 (11/28/2014)
-* Fix pattern captures in status output
-* Allow tasks to be scheduled in constructor
+
+## 4.0.0 (11/29/2014)
+* Significant re-write of 3.0.x branch 
+
+## 3.0.6 
+* Connection failover for clients

data/README.md

@@ -0,0 +1,111 @@
+# gearman-ruby
+
+![Travis CI Build Status](https://travis-ci.org/johnewart/gearman-ruby.svg)
+
+## What is this?
+
+This is a pure-Ruby library for the [gearman][Gearman] distributed job system.
+
+## What needs to be done?
+
+More testing, some code cleanup.
+
+## What's in this?
+
+Right now, this library has both client and worker support for Ruby apps.
+
+## Getting Started
+
+### Client
+
+A very simple client that submits a "sleep" job and waits for 100 seconds for results:
+
+``` ruby
+require 'rubygems'
+require 'gearman'
+
+servers = ['localhost:4730', 'localhost:4731']
+
+client = Gearman::Client.new(servers)
+taskset = Gearman::TaskSet.new(client)
+
+task = Gearman::Task.new('sleep', 20)
+task.on_complete {|d| puts d }
+
+taskset.add_task(task)
+taskset.wait(100)
+```
+
+### Worker
+
+A worker that will process jobs in the 'sleep' queue:
+
+``` ruby
+require 'rubygems'
+require 'logger'
+require 'gearman'
+
+servers = ['localhost:4730']
+
+w = Gearman::Worker.new(servers)
+logger = Logger.new(STDOUT)
+
+# Add a handler for a "sleep" function that takes a single argument, the
+# number of seconds to sleep before reporting success.
+w.add_ability("sleep") do |data,job|
+ seconds = 10
+ logger.info "Sleeping for #{seconds} seconds"
+ (1..seconds.to_i).each do |i|
+   sleep 1
+   # Report our progress to the job server every second.
+   job.report_status(i, seconds)
+ end
+ # Report success.
+ true
+end
+
+loop { w.work }
+```
+
+[gearman]: http://gearman.org
+
+## Authors
+
+* John Ewart <john@johnewart.net> (current maintainer, author of re-write)
+
+<<<<<<< HEAD
+## Contributors (past and present)
+
+* Kim Altintop
+=======
+## Contributors
+
+>>>>>>> New version (4.0) -- substantial rewrite
+* Josh Black (raskchanky)
+* Colin Curtin (perplexes)
+* Brian Cobb (bcobb)
+* Pablo A. Delgado (pablete)
+<<<<<<< HEAD
+* Daniel Erat
+* Antonio Garrote 
+* Stefan Kaes (skaes)
+* Ladislav Martincik
+=======
+* Stefan Kaes (skaes)
+>>>>>>> New version (4.0) -- substantial rewrite
+* Mauro Pompilio (malditogeek)
+* Lee Reilly (leereilly)
+* Clint Shryock (catsby)
+* Andy Triggs (andyt)
+
+
+<<<<<<< HEAD
+
+## License
+
+Released under the MIT license, originally developed by XING AG. See the LICENSE file for further details.
+=======
+## License
+
+Released under the MIT license. See the file LICENSE for further details.
+>>>>>>> New version (4.0) -- substantial rewrite

data/examples/client.rb

@@ -1,7 +1,6 @@
+$LOAD_PATH.unshift("../lib")
 require 'rubygems'
-#require 'gearman'
 require '../lib/gearman'
-# Gearman::Util.debug = true
 
 servers = ['localhost:4730', 'localhost:4731']
   

data/examples/client_reverse_nohost.rb

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require '../lib/gearman'
+l = Logger.new($stdout)
+l.level = Logger::DEBUG
+Gearman::Util.logger=l
+
+# Client using Gearman SUBMIT_JOB_EPOCH (currently requires the gearmand branch lp:~jewart/gearmand/scheduled_jobs_support/)
+
+t = nil
+threadcounter = 0
+
+client = Gearman::Client.new('192.168.1.1:4730')
+
+  
+myid = threadcounter 
+threadcounter += 1
+taskset = Gearman::TaskSet.new(client)
+  
+(1..1000).each do |jid|
+  data = rand(36**8).to_s(36)
+  result = data.reverse
+
+  task = Gearman::BackgroundTask.new("reverse_string", data)
+  puts "#{jid} #{data}"
+        
+  #time = Time.now() + rand(120) + 10
+  #task.schedule(time)
+  taskset.add_task(task)
+end

data/examples/{client_reverse.rb → client_reverse_wait.rb}

@@ -1,27 +1,25 @@
 #!/usr/bin/env ruby
-require 'rubygems'
-require '../lib/gearman'
+$:.unshift File.join(File.dirname(__FILE__), '..', "lib" )
+require 'gearman'
 
 # Client using Gearman SUBMIT_JOB_EPOCH (currently requires the gearmand branch lp:~jewart/gearmand/scheduled_jobs_support/)
 
 t = nil
 threadcounter = 0
 
-client = Gearman::Client.new('localhost')
+client = Gearman::Client.new('localhost:4730')
 
   
 myid = threadcounter 
 threadcounter += 1
 taskset = Gearman::TaskSet.new(client)
   
-(1..10000).each do |jid|
+(1..100).each do |jid|
   data = rand(36**8).to_s(36)
-  result = data.reverse
-
-  task = Gearman::Task.new("reverse_string", data)
   puts "#{jid} #{data}"
-        
-  time = Time.now() + rand(120) + 10
-  task.schedule(time)
-  taskset.add_task(task)
+  task = Gearman::Task.new("reverse_string", data)
+  task.on_complete {|d| puts d }
+  client.do_task(task)
 end
+
+

data/examples/worker.rb

@@ -1,21 +1,24 @@
+$LOAD_PATH.unshift("../lib")
 require 'rubygems'
-#require 'gearman'
+require 'logger'
 require '../lib/gearman'
+servers = ['localhost:4730']
 
-servers = ['localhost:4730',]
 w = Gearman::Worker.new(servers)
+logger = Logger.new(STDOUT)
 
 # Add a handler for a "sleep" function that takes a single argument, the
 # number of seconds to sleep before reporting success.
-w.add_ability('sleep') do |data,job|
- seconds = data
+w.add_ability("sleep") do |data,job|
+ seconds = 10
+ logger.info "Sleeping for #{seconds} seconds"
  (1..seconds.to_i).each do |i|
    sleep 1
-   print i
    # Report our progress to the job server every second.
    job.report_status(i, seconds)
  end
  # Report success.
  true
 end
+
 loop { w.work }

data/examples/worker_reverse_string.rb

@@ -1,27 +1,22 @@
-require 'rubygems'
-require '../lib/gearman'
+$:.unshift File.join(File.dirname(__FILE__), '..', "lib" )
+require 'gearman'
 
 # String reverse worker 
 
-servers = ['localhost:4730']
+servers = ['127.0.0.1:4730']
 
 t = nil
 jobnum = 0
 
-(0..1).each do 
-  t = Thread.new {
-    w = Gearman::Worker.new(servers)
-    w.add_ability('reverse_string') do |data,job|
-      result = data.reverse
-      puts "Job: #{job.inspect} Data: #{data.inspect} Reverse: #{result} "
-      puts "Completed job ##{jobnum}"
-      jobnum += 1
-      result
-    end
-    loop { w.work }
-  }
+w = Gearman::Worker.new(servers)
+w.add_ability('reverse_string') do |data,job|
+   result = data.reverse
+   puts "Job: #{job.inspect} Data: #{data.inspect} Reverse: #{result} "
+   puts "Completed job ##{jobnum}"
+   jobnum += 1
+   result
 end
 
-puts "Waiting for threads..."
-t.join
+loop { w.work }
+
 

data/gearman-ruby.gemspec

@@ -5,8 +5,8 @@ Gem::Specification.new do |s|
   s.name          = %q{gearman-ruby}
   s.version       = Gearman::VERSION
   s.platform      = Gem::Platform::RUBY
-  s.authors       = ["John Ewart", "Colin Curtin", "Daniel Erat", "Ladislav Martincik", "Pablo Delgado", "Mauro Pompilio", "Antonio Garrote", "Kim Altintop"]
-  s.date          = %q{2013-07-25}
+  s.authors       = ["John Ewart"]
+  s.date          = %q{2014-11-29}
   s.summary       = %q{Ruby Gearman library}
   s.description   = %q{Library for the Gearman distributed job system}
   s.email         = %q{john@johnewart.net}
@@ -15,14 +15,12 @@ Gem::Specification.new do |s|
 
   s.extra_rdoc_files = [
     "LICENSE",
-    "README",
-    "TODO"
+    "README.md"
   ]
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
-  s.require_paths = ["lib"]
 end
 

data/lib/gearman.rb

@@ -1,82 +1,22 @@
 #!/usr/bin/env ruby
-#
-# = Name
-# Gearman
-#
-# == Description
-# This file provides a Ruby interface for communicating with the Gearman
-# distributed job system.
-#
-# "Gearman is a system to farm out work to other machines, dispatching
-# function calls to machines that are better suited to do work, to do work
-# in parallel, to load balance lots of function calls, or to call functions
-# between languages."  -- http://www.danga.com/gearman/
-#
-# == Version
-# 0.0.1
-#
-# == Author
-# Daniel Erat <dan-ruby@erat.org>
-#
-# == License
-# This program is free software; you can redistribute it and/or modify it
-# under the terms of either:
-#
-# a) the GNU General Public License as published by the Free Software
-#    Foundation; either version 1, or (at your option) any later version,
-#    or
-#
-# b) the "Artistic License" which comes with Perl.
 
-# = Gearman
-#
-# == Usage
-#  require 'gearman'
-#
-#  # Create a new client and tell it about two job servers.
-#  c = Gearman::Client.new
-#  c.job_servers = ['127.0.0.1:7003', '127.0.0.1:7004']
-#
-#  # Create two tasks, using an "add" function to sum two numbers.
-#  t1 = Gearman::Task.new('add', '5 + 2')
-#  t2 = Gearman::Task.new('add', '1 + 3')
-#
-#  # Make the tasks print the data they get back from the server.
-#  t1.on_complete {|d| puts "t1 got #{d}" }
-#  t2.on_complete {|d| puts "t2 got #{d}" }
-#
-#  # Create a taskset, add the two tasks to it, and wait until they finish.
-#  ts = Gearman::TaskSet.new(c)
-#  ts.add_task(t1)
-#  ts.add_task(t2)
-#  ts.wait
-#
-# Or, a more simple example:
-#
-#  c = Gearman::Client.new('127.0.0.1')
-#  puts c.do_task('add', '2 + 2')
-#
-module Gearman
-
-require File.dirname(__FILE__) + '/gearman/client'
-require File.dirname(__FILE__) + '/gearman/task'
-require File.dirname(__FILE__) + '/gearman/taskset'
-require File.dirname(__FILE__) + '/gearman/util'
-require File.dirname(__FILE__) + '/gearman/worker'
-
-class InvalidArgsError < Exception
-end
-
-class ProtocolError < Exception
-end
+require 'logger'
 
-class NetworkError < Exception
-end
-
-class NoJobServersError < Exception
-end
-
-class JobQueueError < Exception
+module Gearman
+  class << self
+    attr_writer :logger
+    def logger
+      @logger ||= Logger.new(STDOUT)
+    end
+  end
 end
 
-end
+require 'gearman/exceptions'
+require 'gearman/logging'
+require 'gearman/packet'
+require 'gearman/connection'
+require 'gearman/connection_pool'
+require 'gearman/client'
+require 'gearman/task'
+require 'gearman/task_set'
+require 'gearman/worker'

data/lib/gearman/client.rb

@@ -1,169 +1,151 @@
-require 'socket'
+require 'time'
 
 module Gearman
+  class Client
+    include Logging
+
+    attr_accessor :task_create_timeout_sec
+
+    ##
+    # Create a new client.
+    #
+    # @param job_servers  "host:port"; either a single server or an array
+    def initialize(job_servers=nil)
+      @coalesce_connections = {}  # Unique ID => Connection
+      @connection_pool = ConnectionPool.new(job_servers)
+      @current_task = nil
+      @task_create_timeout_sec = 10
+    end
 
-# = Client
-#
-# == Description
-# A client for communicating with Gearman job servers.
-class Client
-  ##
-  # Create a new client.
-  #
-  # @param job_servers  "host:port"; either a single server or an array
-  def initialize(job_servers=nil)
-    @job_servers = []  # "host:port"
-    self.job_servers = job_servers if job_servers
-    @sockets = {}  # "host:port" -> [sock1, sock2, ...]
-    @socket_to_hostport = {}  # sock -> "host:port"
-    @task_create_timeout_sec = 10
-    @server_counter = -1
-    @bad_servers = []
-  end
-  attr_reader :job_servers, :bad_servers
-  attr_accessor :task_create_timeout_sec
-
-  ##
-  # Set the options
-  #
-  # @options options to pass to the servers  "exeptions"
-  def option_request(opts)
-    Util.logger.debug "GearmanRuby: Send options request with #{opts}"
-    request = Util.pack_request("option_req", opts)
-    sock= self.get_socket(self.get_job_server)
-    Util.send_request(sock, request)
-    response = Util.read_response(sock, 20)
-    raise ProtocolError, response[1] if response[0]==:error
-  end
-
-  ##
-  # Set the job servers to be used by this client.
-  #
-  # @param servers  "host:port"; either a single server or an array
-  def job_servers=(servers)
-    @job_servers = Util.normalize_job_servers(servers)
-    self
-  end
-
-  ##
-  # Get connection info about an arbitrary (currently random, but maybe
-  # we'll do something smarter later) job server.
-  #
-  # @return  "host:port"
-  def get_job_server
-    if @job_servers.empty? && !@bad_servers.empty?
-      Util.logger.debug "GearmanRuby: No more good job servers, trying bad ones: #{@bad_servers.inspect}."
-      # Try to reconnect to the bad servers
-      @bad_servers.each do |bad_server|
-        Util.logger.debug "GearmanRuby: Trying server: #{bad_server.inspect}"
-        begin
-          request = Util.pack_request("echo_req", "ping")
-          sock = self.get_socket(bad_server)
-          Util.send_request(sock, request)
-          response = Util.read_response(sock, 20)
-          if response[0] == :echo_res
-            @job_servers << bad_server
-            @bad_servers.delete bad_server
-          end
-        rescue NetworkError
-          Util.logger.debug "GearmanRuby: Error trying server: #{bad_server.inspect}"
-        end
+    ##
+    # Set the options
+    #
+    # @options options to pass to the servers  i.e "exceptions"
+    def set_options(opts)
+      @connection_pool.with_all_connections do |conn|
+        logger.debug "Send options request with #{opts}"
+        request = Packet.pack_request("option_req", opts)
+        response = conn.send_request(request)
+        raise ProtocolError, response[1] if response[0]==:error
       end
     end
 
-    Util.logger.debug "GearmanRuby: job servers: #{@job_servers.inspect}"
-    raise NoJobServersError if @job_servers.empty?
-    @server_counter += 1
-    @job_servers[@server_counter % @job_servers.size]
-  end
+    ##
+    # Perform a single task.
+    #
+    # @param args  A Task to complete
+    # @return      output of the task, or nil on failure
+    def do_task(task)
 
-  def signal_bad_server(hostport)
-    @job_servers = @job_servers.reject { |s| s == hostport }
-    @bad_servers << hostport
-  end
+      result = nil
+      failed = false
+
+      task.on_complete {|v| result = v }
+      task.on_fail { failed = true }
+
+      task_set = TaskSet.new(self)
+      if task_set.add_task(task)
+        task_set.wait_forever
+      else
+        raise JobQueueError, "Unable to enqueue job."
+      end
 
-  ##
-  # Get a socket for a job server.
-  #
-  # @param hostport  job server "host:port"
-  # @return          a Socket
-  def get_socket(hostport, num_retries=3)
-    # If we already have an open socket to this host, return it.
-    if @sockets[hostport]
-      sock = @sockets[hostport].shift
-      @sockets.delete(hostport) if @sockets[hostport].size == 0
-      return sock
+      failed ? nil : result
     end
 
-    num_retries.times do |i|
+    def submit_job(task, reset_state = false, timeout = nil)
+      task.reset_state if reset_state
+      req = task.get_submit_packet()
+      req_timeout = timeout || task_create_timeout_sec
+      # Target the same job manager when submitting jobs
+      # with the same unique id so that we can coalesce
+      coalesce_key = task.get_uniq_hash
+
+      end_time = if timeout
+                   Time.now.to_f + timeout
+                 else
+                   nil
+                 end
+
       begin
-        sock = TCPSocket.new(*hostport.split(':'))
-      rescue Exception
-        # Swallow error so we can retry -> num_retries times
-      else
-        # No error, stash socket mapping and return it
-        @socket_to_hostport[sock] = hostport
-        return sock
+
+        connection = @connection_pool.get_connection(coalesce_key)
+        logger.debug "Using #{connection} to submit job"
+
+        type, data = connection.send_request(req, timeout)
+        logger.debug "Got #{type.to_s} from #{connection}"
+
+        if type == :job_created
+
+          task.handle_created(data)
+
+          if(!task.background)
+            begin
+              remaining = if end_time
+                            (t = end_time - Time.now.to_f) > 0 ? t : 0
+                          else
+                            nil
+                          end
+              type, data = connection.read_response(remaining)
+              handle_response(task, type, data)
+            end while [:work_status, :work_data].include? type
+          end
+
+        else
+          # This shouldn't happen
+          message = "Received #{type.to_s} when we were expecting JOB_CREATED"
+          logger.error message
+          raise ProtocolError, message
+        end
+      rescue NetworkError
+        message = "Network error on read from #{hostport} while adding job, marking server bad"
+        logger.error message
+        raise NetworkError, message
+      rescue NoJobServersError
+        logger.error "No servers available."
+        raise NoJobServersError
       end
+
+      true
     end
-    raise NetworkError, "Unable to connect to job server #{hostport}"
-  end
 
-  ##
-  # Relinquish a socket created by Client#get_socket.
-  #
-  # If we don't know about the socket, we just close it.
-  #
-  # @param sock  Socket
-  def return_socket(sock)
-    hostport = get_hostport_for_socket(sock)
-    if not hostport
-      inet, port, host, ip = sock.addr
-      Util.logger.error "GearmanRuby: Got socket for #{ip}:#{port}, which we don't know about -- closing"
-      sock.close
-      return
+    def handle_response(task, type, data)
+      case type
+        when :work_complete
+          handle, message = data.split("\0", 2)
+          logger.debug("Received WORK_COMPLETE for #{handle}")
+          task.handle_completion(message)
+        when :work_exception
+          handle, exception = data.split("\0", 2)
+          logger.debug("Received WORK_EXCEPTION for #{handle}")
+          task.handle_exception(exception)
+        when :work_fail
+          logger.debug("Received WORK_FAIL for #{handle}")
+          requeue = task.handle_failure
+          add_task(task) if requeue
+        when :work_status
+          handle, numerator, denominator = data.split("\0", 3)
+          logger.debug("Received WORK_STATUS for #{handle}: #{numerator} / #{denominator}")
+          task.handle_status(numerator, denominator)
+        when :work_warning
+          handle, message = data.split("\0", 2)
+          Util.logger.debug "Got WORK_WARNING for #{handle}: '#{message}'"
+          task.handle_warning(message)
+        when :work_data
+          handle, work_data = data.split("\0", 2)
+          Util.logger.debug "Got WORK_DATA for #{handle} with #{work_data ? work_data.size : '0'} byte(s) of data"
+          task.handle_data(work_data)
+        else
+          # Not good.
+          message = "Got #{type.to_s} from #{connection} but it was not an expected type."
+          logger.error message
+          raise ProtocolError, message
+      end
     end
-    (@sockets[hostport] ||= []) << sock
-  end
 
-  def close_socket(sock)
-    sock.close
-    @socket_to_hostport.delete(sock)
-    nil
   end
 
-  ##
-  # Given a socket from Client#get_socket, return its host and port.
-  #
-  # @param sock  Socket
-  # @return      "host:port", or nil if unregistered (which shouldn't happen)
-  def get_hostport_for_socket(sock)
-    @socket_to_hostport[sock]
-  end
 
-  ##
-  # Perform a single task.
-  #
-  # @param args  either a Task or arguments for Task.new
-  # @return      output of the task, or nil on failure
-  def do_task(*args)
-    task = Util::get_task_from_args(*args)
-
-    result = nil
-    failed = false
-    task.on_complete {|v| result = v }
-    task.on_fail { failed = true }
-
-    taskset = TaskSet.new(self)
-    if taskset.add_task(task)
-      taskset.wait
-    else
-      raise JobQueueError, "Unable to enqueue job."
-    end
 
-    failed ? nil : result
-  end
 
 end
-
-end