gearman-ruby 3.0.8 → 4.0.2

data/lib/gearman/taskset.rb

@@ -1,293 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'socket'
-require 'time'
-
-module Gearman
-
-# = TaskSet
-#
-# == Description
-# A set of tasks submitted to a Gearman job server.
-class TaskSet
-  def initialize(client)
-    @client = client
-    @task_waiting_for_handle = nil
-    @tasks_in_progress = {}  # "host:port//handle" -> [job1, job2, ...]
-    @finished_tasks = []  # tasks that have completed or failed
-    @sockets = {}  # "host:port" -> Socket
-    @merge_hash_to_hostport = {}  # Fixnum -> "host:port"
-  end
-
-  ##
-  # Add a new task to this TaskSet.
-  #
-  # @param args  either a Task or arguments for Task.new
-  # @return      true if the task was created successfully, false otherwise
-  def add_task(*args)
-    task = Util::get_task_from_args(*args)
-    add_task_internal(task, true)
-  end
-
-  ##
-  # Internal function to add a task.
-  #
-  # @param task         Task to add
-  # @param reset_state  should we reset task state?  true if we're adding a
-  #                     new task; false if we're rescheduling one that's
-  #                     failed
-  # @return             true if the task was created successfully, false
-  #                     otherwise
-  def add_task_internal(task, reset_state=true)
-    task.reset_state if reset_state
-    req = task.get_submit_packet()
-
-    @task_waiting_for_handle = task
-
-    # Target the same job manager when submitting jobs
-    # with the same unique id so that we can coalesce
-    merge_hash = task.get_uniq_hash
-
-    while (@task_waiting_for_handle != nil)
-      begin
-        # Try to connect again to the same server based on the unique hash
-        @merge_hash_to_hostport[merge_hash] ||= @client.get_job_server
-        hostport = @merge_hash_to_hostport[merge_hash]
-
-        # Cache the socket
-        @sockets[hostport] ||= @client.get_socket(hostport)
-
-        # Submit job to server
-        sock = @sockets[hostport]
-        Util.logger.debug "GearmanRuby: Using socket #{sock.inspect} for #{hostport} to SUBMIT_JOB"
-        Util.send_request(sock, req)
-
-        # read_packet will fire off handle_job_created and set @task_waiting_for_handle to nil
-        # TODO: Better way of doing this.
-        read_packet(sock, @client.task_create_timeout_sec)
-      rescue NetworkError
-        Util.logger.debug "GearmanRuby: Network error on read from #{hostport} while adding job, marking server bad"
-        # Tell the client this is a bad server
-        @client.signal_bad_server(hostport)
-        if(sock != nil)
-          @client.close_socket(sock)
-        end
-
-        # Un-cache socket
-        @sockets.delete hostport
-        # Remove from hash -> hostport mapping
-        @merge_hash_to_hostport[merge_hash] = nil
-      rescue NoJobServersError
-        Util.logger.error "GearmanRuby: No servers available."
-        return false
-      end
-    end
-
-    return true
-  end
-  private :add_task_internal
-
-  ##
-  # Handle a 'job_created' response from a job server.
-  #
-  # @param hostport  "host:port" of job server
-  # @param data      data returned in packet from server
-  def handle_job_created(hostport, data)
-    Util.logger.debug "GearmanRuby: Got job_created with handle #{data} from #{hostport}"
-
-    if not @task_waiting_for_handle
-      raise ProtocolError, "Got unexpected job_created notification with handle #{data} from #{hostport}"
-    end
-
-    js_handle = Util.handle_to_str(hostport, data)
-    task = @task_waiting_for_handle
-    @task_waiting_for_handle = nil
-    if(task.background)
-      @finished_tasks << task
-    else
-      (@tasks_in_progress[js_handle] ||= []) << task
-    end
-    task.handle_created(data)
-    nil
-  end
-  private :handle_job_created
-
-  ##
-  # Handle a 'work_complete' response from a job server.
-  #
-  # @param hostport  "host:port" of job server
-  # @param data      data returned in packet from server
-  def handle_work_complete(hostport, data)
-    handle, data = data.split("\0", 2)
-    Util.logger.debug "GearmanRuby: Got work_complete with handle #{handle} and #{data ? data.size : '0'} byte(s) of data from #{hostport}"
-    tasks_in_progress(hostport, handle, true).each do |t|
-      t.handle_completion(data)
-      @finished_tasks << t
-    end
-    nil
-  end
-  private :handle_work_complete
-
-  ##
-  # Handle a 'work_exception' response from a job server.
-  #
-  # @param hostport  "host:port" of job server
-  # @param data      data returned in packet from server
-  def handle_work_exception(hostport, data)
-    handle, exception = data.split("\0", 2)
-    Util.logger.debug "GearmanRuby: Got work_exception with handle #{handle} from #{hostport}: '#{exception}'"
-    tasks_in_progress(hostport, handle).each {|t| t.handle_exception(exception) }
-  end
-  private :handle_work_exception
-
-  ##
-  # Handle a 'work_fail' response from a job server.
-  #
-  # @param hostport  "host:port" of job server
-  # @param data      data returned in packet from server
-  def handle_work_fail(hostport, data)
-    Util.logger.debug "GearmanRuby: Got work_fail with handle #{data} from #{hostport}"
-    tasks_in_progress(hostport, data, true).each do |t|
-      if t.handle_failure
-        add_task_internal(t, false)
-      else
-        @finished_tasks << t
-      end
-    end
-  end
-  private :handle_work_fail
-
-  ##
-  # Handle a 'work_status' response from a job server.
-  #
-  # @param hostport  "host:port" of job server
-  # @param data      data returned in packet from server
-  def handle_work_status(hostport, data)
-    handle, num, den = data.split("\0", 3)
-    Util.logger.debug "GearmanRuby: Got work_status with handle #{handle} from #{hostport}: #{num}/#{den}"
-    tasks_in_progress(hostport, handle).each {|t| t.handle_status(num, den) }
-  end
-  private :handle_work_status
-
-  ##
-  # Handle a 'work_warning' response from a job server.
-  #
-  # @param hostport "host:port" of job server
-  # @param data     data returned in packet from server
-  def handle_work_warning(hostport, data)
-    handle, message = data.split("\0", 2)
-    Util.logger.debug "GearmanRuby: Got work_warning with handle #{handle} from #{hostport}: '#{message}'"
-    tasks_in_progress(hostport, handle).each {|t| t.handle_warning(message) }
-  end
-  private :handle_work_warning
-
-  ##
-  # Handle a 'work_data' response from a job server
-  #
-  # @param hostport  "host:port" of a job server
-  # @param data       data returned in packet from server
-  def handle_work_data(hostport, data)
-    handle, data = data.split("\0", 2)
-    Util.logger.debug "GearmanRuby: Got work_data with handle #{handle} and #{data ? data.size : '0'} byte(s) of data from #{hostport}"
-
-    js_handle = Util.handle_to_str(hostport, handle)
-    tasks = @tasks_in_progress[js_handle]
-    if not tasks
-      raise ProtocolError, "Got unexpected work_data with handle #{handle} from #{hostport} (no task by that name)"
-    end
-    tasks.each {|t| t.handle_data(data) }
-  end
-  private :handle_work_data
-
-  ##
-  # Read and process a packet from a socket.
-  #
-  # @param sock  socket connected to a job server
-  def read_packet(sock, timeout=nil)
-    hostport = @client.get_hostport_for_socket(sock)
-    if not hostport
-      raise RuntimeError, "Client doesn't know host/port for socket " +
-        sock.inspect
-    end
-    type, data = Util.read_response(sock, timeout)
-    known_types = [ :job_created,
-                    :work_complete,
-                    :work_fail,
-                    :work_status,
-                    :work_exception,
-                    :work_warning,
-                    :work_data ]
-
-    if known_types.include?(type)
-      send("handle_#{type}".to_sym, hostport, data)
-    else
-      Util.logger.debug "GearmanRuby: Got #{type.to_s} from #{hostport}"
-    end
-    nil
-  end
-  private :read_packet
-
-  ##
-  # Wait for all tasks in the set to finish.
-  #
-  # @param timeout  maximum amount of time to wait, in seconds
-  def wait(timeout = 1)
-    end_time = if timeout
-      Time.now.to_f + timeout
-    else
-      nil
-    end
-
-    while not @tasks_in_progress.empty?
-      remaining = if end_time
-        (t = end_time - Time.now.to_f) > 0 ? t : 0
-      else
-        nil
-      end
-
-      ready_socks = remaining == 0 ? nil : IO::select(@sockets.values, nil, nil, remaining)
-      if not ready_socks or not ready_socks[0]
-        Util.logger.debug "GearmanRuby: Timed out while waiting for tasks to finish"
-        # not sure what state the connections are in, so just be lame and
-        # close them for now
-        @sockets.values.each {|s| @client.close_socket(s) }
-        @sockets = {}
-        return false
-      end
-
-      ready_socks[0].each do |sock|
-        begin
-           read_packet(sock, (end_time ? end_time - Time.now.to_f : nil))
-        rescue ProtocolError
-          hostport = @client.get_hostport_for_socket(sock)
-          Util.logger.debug "GearmanRuby: Ignoring bad packet from #{hostport}"
-        rescue NetworkError
-          hostport = @client.get_hostport_for_socket(sock)
-          Util.logger.debug "GearmanRuby: Network error on read from #{hostport}"
-        end
-      end
-    end
-
-    @sockets.values.each {|s| @client.return_socket(s) }
-    @sockets = {}
-    @finished_tasks.each do |t|
-      if ( (t.background.nil? || t.background == false) && !t.successful)
-        Util.logger.debug "GearmanRuby: TaskSet failed"
-        return false
-      end
-    end
-    true
-  end
-
-  private
-    def tasks_in_progress(hostport, handle, remove_task = false)
-      js_handle = Util.handle_to_str(hostport, handle)
-      tasks = remove_task ? @tasks_in_progress.delete(js_handle) : @tasks_in_progress[js_handle]
-      if not tasks
-        raise ProtocolError, "Got unexpected work_data with handle #{handle} from #{hostport} (no task by that name)"
-      end
-      tasks
-    end
-end
-
-end

data/lib/gearman/util.rb

@@ -1,211 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'socket'
-require 'time'
-require 'logger'
-
-module Gearman
-
-  class ServerDownException < Exception; end
-
-# = Util
-#
-# == Description
-# Static helper methods and data used by other classes.
-class Util
-  # Map from Integer representations of commands used in the network
-  # protocol to more-convenient symbols.
-  COMMANDS = {
-    1  => :can_do,               # W->J: FUNC
-    2  => :cant_do,              # W->J: FUNC
-    3  => :reset_abilities,      # W->J: --
-    4  => :pre_sleep,            # W->J: --
-    #5 =>  (unused),             # -      -
-    6  => :noop,                 # J->W: --
-    7  => :submit_job,           # C->J: FUNC[0]UNIQ[0]ARGS
-    8  => :job_created,          # J->C: HANDLE
-    9  => :grab_job,             # W->J: --
-    10 => :no_job,               # J->W: --
-    11 => :job_assign,           # J->W: HANDLE[0]FUNC[0]ARG
-    12 => :work_status,          # W->J/C: HANDLE[0]NUMERATOR[0]DENOMINATOR
-    13 => :work_complete,        # W->J/C: HANDLE[0]RES
-    14 => :work_fail,            # W->J/C: HANDLE
-    15 => :get_status,           # C->J: HANDLE
-    16 => :echo_req,             # ?->J: TEXT
-    17 => :echo_res,             # J->?: TEXT
-    18 => :submit_job_bg,        # C->J: FUNC[0]UNIQ[0]ARGS
-    19 => :error,                # J->?: ERRCODE[0]ERR_TEXT
-    20 => :status_res,           # C->J: HANDLE[0]KNOWN[0]RUNNING[0]NUM[0]DENOM
-    21 => :submit_job_high,      # C->J: FUNC[0]UNIQ[0]ARGS
-    22 => :set_client_id,        # W->J: [RANDOM_STRING_NO_WHITESPACE]
-    23 => :can_do_timeout,       # W->J: FUNC[0]TIMEOUT
-    24 => :all_yours,            # REQ    Worker
-    25 => :work_exception,       # W->J: HANDLE[0]ARG
-    26 => :option_req,           # C->J: TEXT
-    27 => :option_res,           # J->C: TEXT
-    28 => :work_data,            # REQ    Worker
-    29 => :work_warning,         # W->J/C: HANDLE[0]MSG
-    30 => :grab_job_uniq,        # REQ    Worker
-    31 => :job_assign_uniq,      # RES    Worker
-    32 => :submit_job_high_bg,   # C->J: FUNC[0]UNIQ[0]ARGS
-    33 => :submit_job_low,       # C->J: FUNC[0]UNIQ[0]ARGS
-    34 => :submit_job_low_bg,    # C->J: FUNC[0]UNIQ[0]ARGS
-    35 => :submit_job_sched,     # REQ    Client
-    36 => :submit_job_epoch      # C->J: FUNC[0]UNIQ[0]EPOCH[0]ARGS
-  }
-
-  # Map e.g. 'can_do' => 1
-  NUMS = COMMANDS.invert
-
-  # Default job server port.
-  DEFAULT_PORT = 4730
-
-  def Util.logger=(logger)
-    @logger = logger
-  end
-
-  def Util.logger
-    @logger ||=
-      begin
-        l = Logger.new($stdout)
-        l.level = Logger::FATAL
-        l
-      end
-  end
-
-  ##
-  # Construct a request packet.
-  #
-  # @param type_name  command type's name (see COMMANDS)
-  # @param arg        optional data to pack into the command
-  # @return           packet (as a string)
-  def Util.pack_request(type_name, arg='')
-    type_num = NUMS[type_name.to_sym]
-    raise InvalidArgsError, "Invalid type name '#{type_name}'" unless type_num
-    arg = '' if not arg
-    "\0REQ" + [type_num, arg.size].pack('NN') + arg
-  end
-
-  ##
-  # Return a Task based on the passed-in arguments.
-  #
-  # @param args  either a single Task object or the arguments accepted by
-  #              Task.new
-  # @return      Task object
-  def Util.get_task_from_args(*args)
-    if (args[0].class == Task || args[0].class.superclass == Task)
-      return args[0]
-    elsif args.size <= 3
-      return Task.new(*args)
-    else
-      raise InvalidArgsError, 'Incorrect number of args to get_task_from_args'
-    end
-  end
-
-  ##
-  # Read from a socket, giving up if it doesn't finish quickly enough.
-  # NetworkError is thrown if we don't read all the bytes in time.
-  #
-  # @param sock     Socket from which we read
-  # @param len      number of bytes to read
-  # @param timeout  maximum number of seconds we'll take; nil for no timeout
-  # @return         full data that was read
-  def Util.timed_recv(sock, len, timeout=nil)
-    data = ''
-    end_time = Time.now.to_f + timeout if timeout
-    while data.size < len and (not timeout or Time.now.to_f < end_time) do
-      IO::select([sock], nil, nil, timeout ? end_time - Time.now.to_f : nil) \
-        or break
-      begin
-        data += sock.readpartial(len - data.size)
-      rescue
-        raise NetworkError, "Unable to read data from socket."
-      end
-    end
-    if data.size < len
-      raise NetworkError, "Read #{data.size} byte(s) instead of #{len}"
-    end
-    data
-  end
-
-  ##
-  # Read a response packet from a socket.
-  #
-  # @param sock     Socket connected to a job server
-  # @param timeout  timeout in seconds, nil for no timeout
-  # @return         array consisting of integer packet type and data
-  def Util.read_response(sock, timeout=nil)
-    end_time = Time.now.to_f + timeout if timeout
-    head = timed_recv(sock, 12, timeout)
-    magic, type, len = head.unpack('a4NN')
-    raise ProtocolError, "Invalid magic '#{magic}'" unless magic == "\0RES"
-    buf = len > 0 ?
-      timed_recv(sock, len, timeout ? end_time - Time.now.to_f : nil) : ''
-    type = COMMANDS[type]
-    raise ProtocolError, "Invalid packet type #{type}" unless type
-    [type, buf]
-  end
-
-  ##
-  # Send a request packet over a socket.
-  #
-  # @param sock  Socket connected to a job server
-  # @param req   request packet to send
-  def Util.send_request(sock, req)
-    len = with_safe_socket_op{ sock.write(req) }
-    if len != req.size
-      raise NetworkError, "Wrote #{len} instead of #{req.size}"
-    end
-  end
-
-  ##
-  # Add default ports to a job server or list of servers.
-  #
-  # @param servers  a server hostname or "host:port" or array of servers
-  # @return         an array of "host:port" strings
-  def Util.normalize_job_servers(servers)
-    if servers.class == String or servers.class == Symbol
-      servers = [ servers.to_s ]
-    end
-    servers.map {|s| s =~ /:/ ? s : "#{s}:#{DEFAULT_PORT}" }
-  end
-
-  ##
-  # Convert job server info and a handle into a string.
-  #
-  # @param hostport  "host:port" of job server
-  # @param handle    job server-returned handle for a task
-  # @return          "host:port//handle"
-  def Util.handle_to_str(hostport, handle)
-    "#{hostport}//#{handle}"
-  end
-
-  ##
-  # Reverse Util.handle_to_str.
-  #
-  # @param str  "host:port//handle"
-  # @return     [hostport, handle]
-  def Util.str_to_handle(str)
-    str =~ %r{^([^:]+:\d+)//(.+)}
-    return [$1, $2]
-  end
-
-  def self.with_safe_socket_op
-    begin
-      yield
-    rescue Exception => ex
-      raise ServerDownException.new(ex.message)
-    end
-  end
-
-  def Util.ability_name_with_prefix(prefix,name)
-    "#{prefix}\t#{name}"
-  end
-
-  class << self
-    alias :ability_name_for_perl :ability_name_with_prefix
-  end
-
-end
-
-end