evented-gearman-ruby 1.0.0

data/lib/gearman/evented/reactor.rb

@@ -0,0 +1,86 @@
+module Gearman
+  module Evented
+
+    module Reactor
+      include EM::Deferrable
+
+      def self.included(mod)
+        mod.instance_eval do
+          def connect(host, port, opts = {})
+            Gearman::Evented::Reactor.connect(host, port, self, opts)
+          end
+        end
+      end
+
+      def self.connect(host, port, reactor, opts = {})
+        EM.connect(host, (port || 4730), reactor) do |c|
+          c.instance_eval do
+            @host = host
+            @port = port || 4730
+            @opts = opts
+          end
+        end
+      end
+
+      def connected?
+        @connected
+      end
+
+      def server
+        @hostport ||= [ @host, @port ].join(":")
+      end
+
+      def connection_completed
+        log "connected to #{@host}:#{@port}"
+        @connected = true
+        @reconnecting = false
+        @reconnect = true
+        succeed
+      end
+
+      def unbind
+        log "disconnected from #{@host}:#{@port}"
+        @connected = false
+        EM.next_tick { reconnect } unless !@reconnect
+        @reconnect = true
+      end
+
+      def disconnect
+        log "force disconnect from #{@host}:#{@port}"
+        @reconnect = false
+        close_connection_after_writing
+      end
+
+      def reconnect(force = false)
+        if @reconnecting
+          EM.add_timer(@opts[:reconnect_sec] || 30) { reconnect }
+          return
+        elsif !@reconnect && !force
+          log "forced disconnect, aborting reconnect attempt"
+          @reconnect = true
+          return
+        else
+          @reconnecting = true
+          @deferred_status = nil
+        end
+
+        log "reconnecting to #{@host}:#{@port}"
+        EM.reconnect(@host, @port.to_i, self)
+      end
+
+      def send(command, data = nil)
+        log "send #{command} #{data.inspect} (#{server})"
+        send_data(Gearman::Protocol.encode_request(command, data))
+      end
+
+      def log(msg, force = false)
+        Gearman::Util.log(msg, force)
+      end
+
+      def to_s
+        "#{@host}:#{@port}"
+      end
+    end
+
+  end
+end

data/lib/gearman/evented/worker.rb

@@ -0,0 +1,118 @@
+module Gearman
+  module Evented
+
+    module WorkerReactor
+      include Gearman::Evented::Reactor
+
+      def connection_completed
+        send :set_client_id, client_id
+        super
+
+        @abilities ||= @opts.delete(:abilities) || []
+        @abilities.each do |ability, args|
+          announce_ability(ability, args[:timeout])
+        end
+
+        grab_job
+      end
+
+      def announce_ability(name, timeout)
+        cmd = timeout ? :can_do_timeout : :can_do
+        arg = timeout ? "#{name}\0#{timeout.to_s}" : name
+        log "announce_ability #{name} #{timeout}"
+        send cmd, arg
+      end
+
+      def announce_disability(name)
+        send :cant_do, name
+      end
+
+      def grab_job
+        log "Grab Job"
+        send :grab_job_uniq
+      end
+
+      def work_fail(handle)
+        send :work_fail, handle
+      end
+
+      def work_complete(handle, data)
+        send :work_complete, "#{handle}\0#{data}"
+      end
+
+      def work_warning(handle, message)
+        send :work_warning, "#{handle}\0#{message}"
+      end
+
+      def receive_data(data)
+        Gearman::Protocol.decode_response(data).each do |type, handle, *data|
+          dispatch_packet(type, handle, *data)
+        end
+      end
+
+      def dispatch_packet(type, handle, *data)
+        success = true
+        timer   = 0
+        case type
+        when :no_job
+          send :pre_sleep
+          timer = @opts[:reconnect_sec] || 30
+        when :job_assign, :job_assign_uniq
+          log "job assign #{handle}, #{data.inspect}"
+          handle_job_assign(handle, data[0], data[1])
+        when :noop
+          log "NOOP"
+        when :error
+          log "[ERROR]: error from server #{server}: #{data}"
+        else
+          log "Got unknown #{type}, #{data} from #{server}"
+        end
+
+        EM.add_timer(timer) { grab_job }
+        succeed [handle, data]
+      end
+
+      def handle_job_assign(handle, func, args = '')
+        return unless handle
+        unless func
+          log "ERROR: Ignoring job_assign with no function"
+          return
+        end
+
+        log "Got job_assign '#{func}' with handle #{handle} and #{args.size rescue 0} byte(s)"
+
+        unless @abilities.has_key?(func)
+          log "Ignoring job_assign for unsupported func #{func} with handle #{handle}"
+          work_fail handle
+          return
+        end
+
+        exception = nil
+        begin
+          ret = @abilities[func][:callback].call(args, Gearman::Job.new(self, handle))
+        rescue Exception => e
+          exception = e
+        end
+
+        if ret && exception.nil?
+          ret = ret.to_s
+          log "Sending work_complete for #{handle} with #{ret.size} byte(s)"
+          work_complete handle, ret
+        elsif exception.nil?
+          log "Sending work_fail for #{handle} to #{server}"
+          work_fail handle
+        elsif exception
+          log "exception #{exception.message}, sending work_warning, work_fail for #{handle}"
+          work_warning handle, exception.message
+          work_fail handle
+        end
+      end
+
+      def client_id
+        @client_id ||= `uuidgen`.strip
+      end
+
+    end
+
+  end
+end

data/lib/gearman/job.rb

@@ -0,0 +1,38 @@
+module Gearman
+  # = Job
+  #
+  # == Description
+  # Interface to allow a worker to report information to a job server.
+  class Job
+    ##
+    # Create a new Job.
+    #
+    # @param sock    Socket connected to job server
+    # @param handle  job server-supplied job handle
+    def initialize(client, handle)
+      @client = client
+      @handle = handle
+    end
+    ##
+    # Report our status to the job server.
+    def report_status(numerator, denominator)
+      @client.send :work_status, "#{@handle}\0#{numerator}\0#{denominator}"
+      self
+    end
+
+    ##
+    # Send data before job completes
+    def send_partial(data)
+      @client.send :work_data, "#{@handle}\0#{data}"
+      self
+    end
+    alias :send_data :send_partial
+
+    ##
+    # Send a warning explicitly
+    def report_warning(warning)
+      @client.send :work_warning, "#{@handle}\0#{warning}"
+      self
+    end
+  end
+end

data/lib/gearman/protocol.rb

@@ -0,0 +1,110 @@
+module Gearman
+
+  class ProtocolError < Exception; end
+
+
+  class Protocol
+    # Map from Integer representations of commands used in the network
+    # protocol to more-convenient symbols.
+    COMMANDS = {
+      1  => :can_do,               # W->J: FUNC
+      2  => :cant_do,              # W->J: FUNC
+      3  => :reset_abilities,      # W->J: --
+      4  => :pre_sleep,            # W->J: --
+      #5 =>  (unused),             # -      -
+      6  => :noop,                 # J->W: --
+      7  => :submit_job,           # C->J: FUNC[0]UNIQ[0]ARGS
+      8  => :job_created,          # J->C: HANDLE
+      9  => :grab_job,             # W->J: --
+      10 => :no_job,               # J->W: --
+      11 => :job_assign,           # J->W: HANDLE[0]FUNC[0]ARG
+      12 => :work_status,          # W->J/C: HANDLE[0]NUMERATOR[0]DENOMINATOR
+      13 => :work_complete,        # W->J/C: HANDLE[0]RES
+      14 => :work_fail,            # W->J/C: HANDLE
+      15 => :get_status,           # C->J: HANDLE
+      16 => :echo_req,             # ?->J: TEXT
+      17 => :echo_res,             # J->?: TEXT
+      18 => :submit_job_bg,        # C->J: FUNC[0]UNIQ[0]ARGS
+      19 => :error,                # J->?: ERRCODE[0]ERR_TEXT
+      20 => :status_res,           # C->J: HANDLE[0]KNOWN[0]RUNNING[0]NUM[0]DENOM
+      21 => :submit_job_high,      # C->J: FUNC[0]UNIQ[0]ARGS
+      22 => :set_client_id,        # W->J: [RANDOM_STRING_NO_WHITESPACE]
+      23 => :can_do_timeout,       # W->J: FUNC[0]TIMEOUT
+      24 => :all_yours,            # REQ    Worker
+      25 => :work_exception,       # W->J: HANDLE[0]ARG
+      26 => :option_req,           # C->J: TEXT
+      27 => :option_res,           # J->C: TEXT
+      28 => :work_data,            # REQ    Worker
+      29 => :work_warning,         # W->J/C: HANDLE[0]MSG
+      30 => :grab_job_uniq,        # REQ    Worker
+      31 => :job_assign_uniq,      # RES    Worker
+      32 => :submit_job_high_bg,   # C->J: FUNC[0]UNIQ[0]ARGS
+      33 => :submit_job_low,       # C->J: FUNC[0]UNIQ[0]ARGS
+      34 => :submit_job_low_bg,    # C->J: FUNC[0]UNIQ[0]ARGS
+      35 => :submit_job_sched,     # REQ    Client
+      36 => :submit_job_epoch      # REQ    Client
+    }
+
+    # Map e.g. 'can_do' => 1
+    COMMANDS_NUMERIC = COMMANDS.invert
+
+    # Default job server port.
+    DEFAULT_PORT = 4730
+
+    class << self
+
+      def encode_request(type_name, arg = nil)
+        type_num = COMMANDS_NUMERIC[type_name.to_sym]
+        raise InvalidArgsError, "Invalid type name '#{type_name}'" unless type_num
+        arg = '' if not arg
+        "\0REQ" + [type_num, arg.size].pack('NN') + arg
+      end
+
+      def decode_response(data, packets = [])
+        # p data
+        magic, type, len = data[0..12].unpack('a4NN')
+        raise ProtocolError, "Invalid magic '#{magic}'" unless magic == "\0RES"
+        type = COMMANDS[type]
+        raise ProtocolError, "Invalid packet type #{type}" unless type
+
+        buf, *more = data[12, data.size].split("\0RES")
+        handle, data = buf ? buf.split("\0", 2) : nil
+        packets << response_packet(type, handle, data)
+        more.each do |packet|
+          decode_response("\0RES#{packet}", packets)
+        end
+        packets
+      end
+
+      def response_packet(type, handle, data)
+        case type
+        when :work_complete,
+             :work_exception,
+             :work_warning,
+             :work_data,
+             :error
+          [type, handle, data]
+        when :job_assign
+          func, data = data.split("\0", 3)
+          [type, handle, func.to_s, data]
+        when :job_assign_uniq
+          func, uuid, data = data.split("\0", 3)
+          [type, handle, func.to_s, data, uuid]
+        when :work_fail,
+             :job_created,
+             :no_job,
+             :noop
+          [type, handle]
+        when :work_status
+          num, den = data.split("\0", 3)
+          [type, handle, num, den]
+        when :status_res
+          known, running, num, den = data.split("\0", 4)
+          [type, handle, known, running, num, den]
+        else
+          raise ProtocolError, "Invalid packet #{type}"
+        end
+      end
+    end
+  end
+end

data/lib/gearman/server.rb

@@ -0,0 +1,94 @@
+#!/usr/bin/env ruby
+
+require 'socket'
+require 'gearman'
+
+module Gearman
+
+# = Server
+#
+# == Description
+# A client for managing Gearman job servers.
+class Server
+  ##
+  # Create a new client.
+  #
+  # @param job_servers  "host:port"; either a single server or an array
+  # @param prefix       function name prefix (namespace)
+  def initialize(hostport)
+    @hostport = hostport  # "host:port"
+  end
+  attr_reader :hostport
+
+  ##
+  # Get a socket for a job server.
+  #
+  # @param hostport  job server "host:port"
+  # @return          a Socket
+  def socket(num_retries=3)
+    return @socket if @socket
+    num_retries.times do
+      begin
+        sock = TCPSocket.new(*hostport.split(':'))
+      rescue Exception
+      else
+        return @socket = sock
+      end
+    end
+    raise RuntimeError, "Unable to connect to job server #{hostport}"
+  end
+
+  ##
+  # Sends a command to the server.
+  #
+  # @return a response string
+  def send_command(name)
+    response = ''
+    socket.puts(name)
+    while true do
+      if buf = socket.recv_nonblock(65536) rescue nil
+        response << buf
+        return response if response =~ /\n.\n$/
+      end
+    end
+  end
+
+  ##
+  # Returns results of a 'status' command.
+  #
+  # @return a hash of abilities with queued, active and workers keys.
+  def status
+    status = {}
+    if response = send_command('status')
+      response.split("\n").each do |line|
+        if line.match /^([A-Za-z_]+)\t([A-Za-z_]+)\t(\d+)\t(\d+)\t(\d+)$/
+          (status[$1] ||= {})[$2] = { :queue => $3, :active => $4, :workers => $5 }
+        end
+      end
+    end
+    status
+  end
+
+  ##
+  # Returns results of a 'workers' command.
+  #
+  # @return an array of worker hashes, containing host, status and functions keys.
+  def workers
+    workers = []
+    if response = send_command('workers')
+      response.split("\n").each do |line|
+        if line.match /^(\d+)\s([a-z0-9\:\.]+)\s([^\s]*)\s:\s([a-z_\s\t]+)$/
+          func_parts = $4.split(' ')
+          functions = []
+          while !func_parts.empty?
+            functions << func_parts.shift << '.' << func_parts.shift
+          end
+          workers << { :host => $2, :status => $3, :functions => functions }
+        end
+      end
+    end
+    workers
+  end
+end
+
+end