gearman-ruby 2.0.0

data/lib/gearman/evented/client.rb

@@ -0,0 +1,99 @@
+module Gearman
+  module Evented
+
+    module ClientReactor
+      include Gearman::Evented::Reactor
+
+      def keep_connected
+        @keep_connected ||= (@opts[:keep_connected] || false)
+      end
+
+      def keep_connected=(keep)
+        @keep_connected = keep
+      end
+
+      def connection_completed
+        @cbs_job_created ||= []
+        @pending_jobs    = []
+        @assigned_jobs   = {}
+        @background_jobs = {}
+        super
+      end
+
+      def receive_data(data)
+        packets = Gearman::Protocol.decode_response(data)
+        log "received #{packets.size} packet(s) at once"
+        log "packets: #{packets.inspect}"
+        packets.each do |type, handle, *data|
+          dispatch_packet(type, handle, *data)
+        end
+      end
+
+      def dispatch_packet_callback(&callback)
+        @dispatch_packet_callback = callback
+      end
+
+      def dispatch_packet(type, handle, *data)
+        log "Got #{type.to_s}, #{handle}, #{data.inspect} from #{server}"
+        if type == :job_created
+          job_created(handle)
+          if cb = @cbs_job_created.shift
+            cb.call(handle)
+          end
+        else
+          dispatch(type, handle, data)
+        end
+      end
+
+      def submit_job(task, &cb_job_created)
+        cmd = "submit_job"
+        cmd << "_#{task.priority}" if [ :high, :low ].include?(task.priority)
+        cmd << "_bg" if task.background
+
+        log "#{cmd} #{task.name}, #{task.payload} to #{server}"
+        send cmd.to_sym, [ task.name, task.hash, task.payload ].join("\0")
+        @pending_jobs << task
+        @cbs_job_created << cb_job_created if cb_job_created
+      end
+
+      def job_created(handle)
+        job = @pending_jobs.shift
+        raise ProtocolError, "No job waiting for handle! (#{handle})" unless job
+        EM.add_periodic_timer(job.poll_status_interval) { get_status(handle) } if job.poll_status_interval
+        if job.background
+          @background_jobs[handle] = job
+        else
+          @assigned_jobs[handle] = job
+        end
+      end
+
+      def get_status(handle)
+        send :get_status, handle
+      end
+
+      def dispatch(type, handle, args)
+        return unless type
+        task = @assigned_jobs[handle]
+        task = @background_jobs[handle] unless task
+        raise ProtocolError, "No task by that name: #{handle}" unless task
+
+        if :work_fail == type && task.should_retry?
+          task.dispatch(:on_retry, task.retries_done)
+          @assigned_jobs.delete(handle)
+          submit_job(task)
+          return
+        end
+
+        if type == :status_res
+          task.dispatch(:on_status, args)
+        else
+          task.dispatch(type.to_s.sub("work", "on"), *args)
+        end
+
+        @assigned_jobs.delete(handle) if [:work_complete, :work_fail].include?(type)
+        disconnect if @assigned_jobs.empty? && !keep_connected
+      end
+    end
+
+  end
+end

data/lib/gearman/evented/reactor.rb

@@ -0,0 +1,86 @@
+module Gearman
+  module Evented
+
+    module Reactor
+      include EM::Deferrable
+
+      def self.included(mod)
+        mod.instance_eval do
+          def connect(host, port, opts = {})
+            Gearman::Evented::Reactor.connect(host, port, self, opts)
+          end
+        end
+      end
+
+      def self.connect(host, port, reactor, opts = {})
+        EM.connect(host, (port || 4730), reactor) do |c|
+          c.instance_eval do
+            @host = host
+            @port = port || 4730
+            @opts = opts
+          end
+        end
+      end
+
+      def connected?
+        @connected
+      end
+
+      def server
+        @hostport ||= [ @host, @port ].join(":")
+      end
+
+      def connection_completed
+        log "connected to #{@host}:#{@port}"
+        @connected = true
+        @reconnecting = false
+        @reconnect = true
+        succeed
+      end
+
+      def unbind
+        log "disconnected from #{@host}:#{@port}"
+        @connected = false
+        EM.next_tick { reconnect } unless !@reconnect
+        @reconnect = true
+      end
+
+      def disconnect
+        log "force disconnect from #{@host}:#{@port}"
+        @reconnect = false
+        close_connection_after_writing
+      end
+
+      def reconnect(force = false)
+        if @reconnecting
+          EM.add_timer(@opts[:reconnect_sec] || 30) { reconnect }
+          return
+        elsif !@reconnect && !force
+          log "forced disconnect, aborting reconnect attempt"
+          @reconnect = true
+          return
+        else
+          @reconnecting = true
+          @deferred_status = nil
+        end
+
+        log "reconnecting to #{@host}:#{@port}"
+        EM.reconnect(@host, @port.to_i, self)
+      end
+
+      def send(command, data = nil)
+        log "send #{command} #{data.inspect} (#{server})"
+        send_data(Gearman::Protocol.encode_request(command, data))
+      end
+
+      def log(msg, force = false)
+        Gearman::Util.log(msg, force)
+      end
+
+      def to_s
+        "#{@host}:#{@port}"
+      end
+    end
+
+  end
+end

data/lib/gearman/evented/worker.rb

@@ -0,0 +1,118 @@
+module Gearman
+  module Evented
+
+    module WorkerReactor
+      include Gearman::Evented::Reactor
+
+      def connection_completed
+        send :set_client_id, client_id
+        super
+
+        @abilities ||= @opts.delete(:abilities) || []
+        @abilities.each do |ability, args|
+          announce_ability(ability, args[:timeout])
+        end
+
+        grab_job
+      end
+
+      def announce_ability(name, timeout)
+        cmd = timeout ? :can_do_timeout : :can_do
+        arg = timeout ? "#{name}\0#{timeout.to_s}" : name
+        log "announce_ability #{name} #{timeout}"
+        send cmd, arg
+      end
+
+      def announce_disability(name)
+        send :cant_do, name
+      end
+
+      def grab_job
+        log "Grab Job"
+        send :grab_job_uniq
+      end
+
+      def work_fail(handle)
+        send :work_fail, handle
+      end
+
+      def work_complete(handle, data)
+        send :work_complete, "#{handle}\0#{data}"
+      end
+
+      def work_warning(handle, message)
+        send :work_warning, "#{handle}\0#{message}"
+      end
+
+      def receive_data(data)
+        Gearman::Protocol.decode_response(data).each do |type, handle, *data|
+          dispatch_packet(type, handle, *data)
+        end
+      end
+
+      def dispatch_packet(type, handle, *data)
+        success = true
+        timer   = 0
+        case type
+        when :no_job
+          send :pre_sleep
+          timer = @opts[:reconnect_sec] || 30
+        when :job_assign, :job_assign_uniq
+          log "job assign #{handle}, #{data.inspect}"
+          handle_job_assign(handle, data[0], data[1])
+        when :noop
+          log "NOOP"
+        when :error
+          log "[ERROR]: error from server #{server}: #{data}"
+        else
+          log "Got unknown #{type}, #{data} from #{server}"
+        end
+
+        EM.add_timer(timer) { grab_job }
+        succeed [handle, data]
+      end
+
+      def handle_job_assign(handle, func, args = '')
+        return unless handle
+        unless func
+          log "ERROR: Ignoring job_assign with no function"
+          return
+        end
+
+        log "Got job_assign '#{func}' with handle #{handle} and #{args.size rescue 0} byte(s)"
+
+        unless @abilities.has_key?(func)
+          log "Ignoring job_assign for unsupported func #{func} with handle #{handle}"
+          work_fail handle
+          return
+        end
+
+        exception = nil
+        begin
+          ret = @abilities[func][:callback].call(args, Gearman::Job.new(self, handle))
+        rescue Exception => e
+          exception = e
+        end
+
+        if ret && exception.nil?
+          ret = ret.to_s
+          log "Sending work_complete for #{handle} with #{ret.size} byte(s)"
+          work_complete handle, ret
+        elsif exception.nil?
+          log "Sending work_fail for #{handle} to #{server}"
+          work_fail handle
+        elsif exception
+          log "exception #{exception.message}, sending work_warning, work_fail for #{handle}"
+          work_warning handle, exception.message
+          work_fail handle
+        end
+      end
+
+      def client_id
+        @client_id ||= `uuidgen`.strip
+      end
+
+    end
+
+  end
+end

data/lib/gearman/job.rb

@@ -0,0 +1,38 @@
+module Gearman
+  # = Job
+  #
+  # == Description
+  # Interface to allow a worker to report information to a job server.
+  class Job
+    ##
+    # Create a new Job.
+    #
+    # @param sock    Socket connected to job server
+    # @param handle  job server-supplied job handle
+    def initialize(client, handle)
+      @client = client
+      @handle = handle
+    end
+    ##
+    # Report our status to the job server.
+    def report_status(numerator, denominator)
+      @client.send :work_status, "#{@handle}\0#{numerator}\0#{denominator}"
+      self
+    end
+
+    ##
+    # Send data before job completes
+    def send_partial(data)
+      @client.send :work_data, "#{@handle}\0#{data}"
+      self
+    end
+    alias :send_data :send_partial
+
+    ##
+    # Send a warning explicitly
+    def report_warning(warning)
+      @client.send :work_warning, "#{@handle}\0#{warning}"
+      self
+    end
+  end
+end

data/lib/gearman/protocol.rb

@@ -0,0 +1,110 @@
+module Gearman
+
+  class ProtocolError < Exception; end
+
+
+  class Protocol
+    # Map from Integer representations of commands used in the network
+    # protocol to more-convenient symbols.
+    COMMANDS = {
+      1  => :can_do,               # W->J: FUNC
+      2  => :cant_do,              # W->J: FUNC
+      3  => :reset_abilities,      # W->J: --
+      4  => :pre_sleep,            # W->J: --
+      #5 =>  (unused),             # -      -
+      6  => :noop,                 # J->W: --
+      7  => :submit_job,           # C->J: FUNC[0]UNIQ[0]ARGS
+      8  => :job_created,          # J->C: HANDLE
+      9  => :grab_job,             # W->J: --
+      10 => :no_job,               # J->W: --
+      11 => :job_assign,           # J->W: HANDLE[0]FUNC[0]ARG
+      12 => :work_status,          # W->J/C: HANDLE[0]NUMERATOR[0]DENOMINATOR
+      13 => :work_complete,        # W->J/C: HANDLE[0]RES
+      14 => :work_fail,            # W->J/C: HANDLE
+      15 => :get_status,           # C->J: HANDLE
+      16 => :echo_req,             # ?->J: TEXT
+      17 => :echo_res,             # J->?: TEXT
+      18 => :submit_job_bg,        # C->J: FUNC[0]UNIQ[0]ARGS
+      19 => :error,                # J->?: ERRCODE[0]ERR_TEXT
+      20 => :status_res,           # C->J: HANDLE[0]KNOWN[0]RUNNING[0]NUM[0]DENOM
+      21 => :submit_job_high,      # C->J: FUNC[0]UNIQ[0]ARGS
+      22 => :set_client_id,        # W->J: [RANDOM_STRING_NO_WHITESPACE]
+      23 => :can_do_timeout,       # W->J: FUNC[0]TIMEOUT
+      24 => :all_yours,            # REQ    Worker
+      25 => :work_exception,       # W->J: HANDLE[0]ARG
+      26 => :option_req,           # C->J: TEXT
+      27 => :option_res,           # J->C: TEXT
+      28 => :work_data,            # REQ    Worker
+      29 => :work_warning,         # W->J/C: HANDLE[0]MSG
+      30 => :grab_job_uniq,        # REQ    Worker
+      31 => :job_assign_uniq,      # RES    Worker
+      32 => :submit_job_high_bg,   # C->J: FUNC[0]UNIQ[0]ARGS
+      33 => :submit_job_low,       # C->J: FUNC[0]UNIQ[0]ARGS
+      34 => :submit_job_low_bg,    # C->J: FUNC[0]UNIQ[0]ARGS
+      35 => :submit_job_sched,     # REQ    Client
+      36 => :submit_job_epoch      # REQ    Client
+    }
+
+    # Map e.g. 'can_do' => 1
+    COMMANDS_NUMERIC = COMMANDS.invert
+
+    # Default job server port.
+    DEFAULT_PORT = 4730
+
+    class << self
+
+      def encode_request(type_name, arg = nil)
+        type_num = COMMANDS_NUMERIC[type_name.to_sym]
+        raise InvalidArgsError, "Invalid type name '#{type_name}'" unless type_num
+        arg = '' if not arg
+        "\0REQ" + [type_num, arg.size].pack('NN') + arg
+      end
+
+      def decode_response(data, packets = [])
+        # p data
+        magic, type, len = data[0..12].unpack('a4NN')
+        raise ProtocolError, "Invalid magic '#{magic}'" unless magic == "\0RES"
+        type = COMMANDS[type]
+        raise ProtocolError, "Invalid packet type #{type}" unless type
+
+        buf, *more = data[12, data.size].split("\0RES")
+        handle, data = buf ? buf.split("\0", 2) : nil
+        packets << response_packet(type, handle, data)
+        more.each do |packet|
+          decode_response("\0RES#{packet}", packets)
+        end
+        packets
+      end
+
+      def response_packet(type, handle, data)
+        case type
+        when :work_complete,
+             :work_exception,
+             :work_warning,
+             :work_data,
+             :error
+          [type, handle, data]
+        when :job_assign
+          func, data = data.split("\0", 3)
+          [type, handle, func.to_s, data]
+        when :job_assign_uniq
+          func, uuid, data = data.split("\0", 3)
+          [type, handle, func.to_s, data, uuid]
+        when :work_fail,
+             :job_created,
+             :no_job,
+             :noop
+          [type, handle]
+        when :work_status
+          num, den = data.split("\0", 3)
+          [type, handle, num, den]
+        when :status_res
+          known, running, num, den = data.split("\0", 4)
+          [type, handle, known, running, num, den]
+        else
+          raise ProtocolError, "Invalid packet #{type}"
+        end
+      end
+    end
+  end
+end