evented-gearman-ruby 1.0.0

data/lib/gearman/task.rb

@@ -0,0 +1,99 @@
+module Gearman
+  class Task
+
+    attr_reader :name, :payload, :retries_done
+    attr_accessor :retries, :priority, :background, :poll_status_interval
+
+    def initialize(name, payload = nil, opts = {})
+      @name       = name.to_s
+      @payload    = payload || ''
+      @priority   = opts.delete(:priority).to_sym rescue nil
+      @background = opts.delete(:background) ? true : false
+
+      @retries_done = 0
+      @retries      = opts.delete(:retries) || 0
+
+      @poll_status_interval = opts.delete(:poll_status_interval)
+      @uniq = opts.has_key?(:uuid) ? opts.delete(:uuid) : `uuidgen`.strip
+    end
+
+    ##
+    # Set a block of code to be executed when this task completes
+    # successfully.  The returned data will be passed to the block.
+    def on_complete(&f)
+      @on_complete = f
+    end
+
+    ##
+    # Set a block of code to be executed when this task fails.
+    def on_fail(&f)
+      @on_fail = f
+    end
+
+    ##
+    # Set a block of code to be executed when this task is retried after
+    # failing.  The number of retries that have been attempted (including the
+    # current one) will be passed to the block.
+    def on_retry(&f)
+      @on_retry = f
+    end
+
+    ##
+    # Set a block of code to be executed when a remote exception is sent by a worker.
+    # The block will receive the message of the exception passed from the worker.
+    # The user can return true for retrying or false to mark it as finished
+    #
+    # NOTE: this is actually deprecated, cf. https://bugs.launchpad.net/gearmand/+bug/405732
+    #
+    def on_exception(&f)
+      @on_exception = f
+    end
+
+    ##
+    # Set a block of code to be executed when we receive a status update for
+    # this task.  The block will receive two arguments, a numerator and
+    # denominator describing the task's status.
+    def on_status(&f)
+      @on_status = f
+    end
+
+    ##
+    # Set a block of code to be executed when we receive a warning from a worker.
+    # It is recommended for workers to send work_warning, followed by work_fail if
+    # an exception occurs on their side. Don't expect this behavior from workers NOT
+    # using this very library ATM, though. (cf. https://bugs.launchpad.net/gearmand/+bug/405732)
+    def on_warning(&f)
+      @on_warning = f
+    end
+
+    ##
+    # Set a block of code to be executed when we receive a (partial) data packet for this task.
+    # The data received will be passed as an argument to the block.
+    def on_data(&f)
+      @on_data = f
+    end
+
+    ##
+    # Record a failure and check whether we should be retried.
+    #
+    # @return  true if we should be resubmitted; false otherwise
+    def should_retry?
+      return false if @retries_done >= @retries
+      @retries_done += 1
+      true
+    end
+
+    def background?
+      background
+    end
+
+    def dispatch(event, *args)
+      callback = instance_variable_get("@#{event}".to_sym)
+      callback.call(*args) if callback
+    end
+
+    def hash
+      @uniq
+    end
+  end
+end

data/lib/gearman/taskset.rb

@@ -0,0 +1,11 @@
+module Gearman
+  class Taskset < ::Array
+
+    def self.create(task_or_taskset)
+      [*task_or_taskset]
+    end
+
+    alias :add :<<
+    alias :add_task :add
+  end
+end

data/lib/gearman/util.rb

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+require 'socket'
+require 'time'
+
+module Gearman
+
+  class ServerDownException < Exception; end
+
+  # = Util
+  #
+  # == Description
+  # Static helper methods and data used by other classes.
+  class Util
+
+    @@debug = false
+
+    ##
+    # Enable or disable debugging output (off by default).
+    #
+    # @param v  print debugging output
+    def Util.debug=(v)
+      @@debug = v
+    end
+
+    ##
+    # Log a message if debugging is enabled.
+    #
+    # @param str  message to log
+    def Util.log(str, force=false)
+      puts "#{Time.now.strftime '%Y-%m-%d %H:%M:%S'} #{str}" if force or @@debug
+    end
+
+    ##
+    # Log a message no matter what.
+    #
+    # @param str  message to log
+    def Util.err(str)
+      log(str, true)
+    end
+
+    def Util.ability_name_with_prefix(prefix,name)
+      "#{prefix}\t#{name}"
+    end
+
+    class << self
+      alias :ability_name_for_perl :ability_name_with_prefix
+    end
+
+  end
+
+end

data/lib/gearman/worker.rb

@@ -0,0 +1,39 @@
+module Gearman
+  class Worker
+
+    attr_reader :abilities
+
+    def initialize(job_servers, opts = {})
+      @reactors = []
+      @abilities = {}
+
+      @job_servers = Array[*job_servers]
+
+      @opts = opts
+    end
+
+    def add_ability(name, timeout = nil, &f)
+      remove_ability(name) if @abilities.has_key?(name)
+      @abilities[name] = { :callback => f, :timeout => timeout }
+    end
+
+    def remove_ability(name)
+      @abilities.delete(name)
+    end
+
+    def has_ability?(name)
+      @abilities.has_key?(name)
+    end
+
+    def work
+      EM.run do
+        @job_servers.each do |hostport|
+          host, port = hostport.split(":")
+          opts = { :abilities => @abilities }.merge(@opts)
+          Gearman::Evented::WorkerReactor.connect(host, port, opts)
+        end
+      end
+    end
+
+  end
+end

data/test/basic_integration_test.rb

@@ -0,0 +1,121 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class BasicIntegrationTest < Test::Unit::TestCase
+
+  def setup
+    start_gearmand
+    @client = Gearman::Client.new("localhost:4730")
+    @worker = Gearman::Worker.new("localhost:4730")
+    Thread.new { EM.run } unless EM.reactor_running?
+  end
+
+  def teardown
+    stop_gearmand
+  end
+
+  def test_ping_job
+    response = nil
+
+    @worker.add_ability("pingpong") {|data, job| "pong" }
+    @worker.work
+
+    task = Gearman::Task.new("pingpong", "ping")
+
+    task.on_complete {|res| response = res }
+    @client.run task
+
+    assert_equal "pong", response
+  end
+
+  def test_exception_in_worker
+    warning_given = nil
+    failed = false
+
+    task = Gearman::Task.new("crash", "doesntmatter")
+
+    @worker.add_ability("crash") {|data, job| raise Exception.new("BOOM!") }
+    @worker.work
+
+    task.on_warning {|warning| warning_given = warning }
+    task.on_fail { failed = true}
+    @client.run task
+
+    assert_not_nil warning_given
+    assert_equal true, failed
+    assert_equal 0, task.retries_done
+  end
+
+  def test_should_be_able_to_retry_on_worker_exception
+    retry_callback_called = false
+    fail_count = 0
+
+    # Gearman::Util.debug = true
+    task = Gearman::Task.new("crash", "doesntmatter", :retries => 1)
+
+    @worker.add_ability("crash") {|data, job| raise Exception.new("BOOM!") }
+    @worker.work
+
+    task.on_retry {|i| retry_callback_called = true }
+    task.on_fail { fail_count += 1 }
+    @client.run task
+
+    assert_equal true, retry_callback_called
+    assert_equal 1, task.retries_done
+    assert_equal 1, fail_count
+  end
+
+  def test_chunked_response
+    chunks_received = 0
+
+    @worker.add_ability("chunked") do |data, job|
+      5.times {|i| job.send_partial("chunk #{i}") }
+    end
+    @worker.work
+
+    task = Gearman::Task.new("chunked")
+    task.on_data do |data|
+      assert_match /^chunk \d/, data
+      chunks_received += 1
+    end
+    @client.run task
+
+    assert_equal 5, chunks_received
+  end
+
+  def test_background
+    status_received = false
+
+    @worker.add_ability("fireandforget") {|data, job| "this goes to /dev/null" }
+    @worker.work
+
+    task = Gearman::Task.new('fireandforget', 'background', :background => true, :poll_status_interval => 0.1)
+    task.on_complete {|d| flunk "on_complete should never be called for a background job!" }
+    task.on_status {|d| status_received = true }
+    @client.run task
+
+    assert_equal true, status_received
+  end
+
+  def test_non_blocking_run
+    @worker.add_ability("foo") {|data, job| "foo: #{data}" }
+    @worker.work
+
+    task1_complete = false
+    task2_complete = false
+
+    task1 = Gearman::Task.new("foo", 1)
+    task1.on_complete {|d| task1_complete = true}
+    @client.run task1, nil, true
+
+    assert_equal false, task1_complete
+
+    task2 = Gearman::Task.new("foo", 2)
+    task2.on_complete {|d| task2_complete = true}
+    @client.run task2, nil, true
+
+    Thread.new { sleep 0.1 }.join
+
+    assert_equal true, task1_complete
+    assert_equal true, task2_complete
+  end
+end

data/test/crash_test.rb

@@ -0,0 +1,69 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+# Gearman::Util.debug = true
+class CrashTest < Test::Unit::TestCase
+
+  def setup
+    teardown_gearmands
+    Thread.new { EM.run } unless EM.reactor_running?
+  end
+
+  def teardown
+    teardown_gearmands
+  end
+
+  def test_worker_should_reconnect_if_gearmand_goes_away
+    start_gearmand
+    worker = Gearman::Worker.new("localhost:4730", :reconnect_sec => 1)
+    worker.add_ability("foo") {|data, job| "noop!" }
+
+    response = nil
+
+    worker.work
+    stop_gearmand
+    start_gearmand
+
+    task = Gearman::Task.new("foo", "ping")
+    task.on_complete {|res| response = res }
+    Gearman::Client.new("localhost:4730").run task
+
+    assert_equal "noop!", response
+    stop_gearmand
+  end
+
+  def test_client_and_worker_should_use_failover_gearmand_if_primary_is_not_available
+    start_gearmand 4731
+
+    worker = Gearman::Worker.new(["localhost:4730", "localhost:4731"], :reconnect_sec => 1)
+    worker.add_ability("foo") {|data, job| "noop!" }
+
+    response = nil
+    worker.work
+    task = Gearman::Task.new("foo", "ping")
+    task.on_complete {|res| response = res }
+    Gearman::Client.new(["localhost:4730", "localhost:4731"]).run task
+
+    assert_equal "noop!", response
+    stop_gearmand 4731
+  end
+
+  def test_client_and_worker_should_switch_to_failover_gearmand_if_primary_goes_down
+    start_gearmand 4730
+    start_gearmand 4731
+
+    worker = Gearman::Worker.new(["localhost:4730", "localhost:4731"], :reconnect_sec => 1)
+    worker.add_ability("foo") {|data, job| "noop!" }
+
+    response = nil
+    worker.work
+
+    stop_gearmand 4730
+
+    task = Gearman::Task.new("foo", "ping")
+    task.on_complete {|res| response = res }
+    Gearman::Client.new(["localhost:4730", "localhost:4731"]).run task
+
+    assert_equal "noop!", response
+    stop_gearmand 4731
+  end
+end

data/test/job_test.rb

@@ -0,0 +1,30 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class JobTest < Test::Unit::TestCase
+
+  def setup
+    @handle = "H:foo:1"
+    @mock_client = mock()
+    @job = Gearman::Job.new(@mock_client, @handle)
+  end
+
+  def test_supports_report_status
+    @mock_client.expects(:send).with(:work_status, [@handle, 1, 5].join("\0"))
+    @job.report_status(1, 5)
+  end
+
+  def test_supports_send_partial
+    @mock_client.expects(:send).with(:work_data, [@handle, "bar"].join("\0"))
+    @job.send_partial("bar")
+  end
+
+  def test_supports_send_data
+    @mock_client.expects(:send).with(:work_data, [@handle, "bar"].join("\0"))
+    @job.send_data("bar")
+  end
+
+  def test_supports_report_warning
+    @mock_client.expects(:send).with(:work_warning, [@handle, "danger"].join("\0"))
+    @job.report_warning("danger")
+  end
+end

data/test/protocol_test.rb

@@ -0,0 +1,132 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class ProtocolTest < Test::Unit::TestCase
+
+  def test_encode_request
+    payload = ["foo", "123", "bar"].join("\0")
+    expected = "\0REQ" + [Gearman::Protocol::COMMANDS_NUMERIC[:submit_job], payload.size].pack("NN") + payload
+    assert_equal expected, Gearman::Protocol.encode_request(:submit_job, payload)
+  end
+
+  def test_decode_response
+    response = "\0RES" + [Gearman::Protocol::COMMANDS_NUMERIC[:work_data], 3].pack("NN") + handle + "\0foo"
+    packets = Gearman::Protocol.decode_response(response)
+    assert_equal 1, packets.size
+    assert_equal [:work_data, "H:wrk.acme:1", "foo"], packets.first
+  end
+
+  def test_decodes_multiple_response_packets
+    response = "\0RES" + [Gearman::Protocol::COMMANDS_NUMERIC[:work_data], 3].pack("NN") + handle + "\0foo"
+    response << "\0RES" + [Gearman::Protocol::COMMANDS_NUMERIC[:work_data], 3].pack("NN") + handle + "\0bar"
+    response << "\0RES" + [Gearman::Protocol::COMMANDS_NUMERIC[:work_data], 3].pack("NN") + handle + "\0baz"
+
+    assert_equal 3, Gearman::Protocol.decode_response(response).size
+  end
+
+  def test_response_packet
+    packet = [:work_data, handle, "foo"]
+    assert_equal packet, Gearman::Protocol.response_packet(*packet)
+  end
+
+  def test_decodes_work_complete
+    data     = "esta complet"
+    response = "\0RES" + [Gearman::Protocol::COMMANDS_NUMERIC[:work_complete], data.size].pack("NN") + [handle, data].join("\0")
+    assert_equal [:work_complete, handle, data], Gearman::Protocol.decode_response(response).first
+  end
+
+  def test_decodes_work_exception
+    data     = "{native perl exception object}"
+    response = "\0RES" + [Gearman::Protocol::COMMANDS_NUMERIC[:work_exception], data.size].pack("NN") + [handle, data].join("\0")
+    assert_equal [:work_exception, handle, data], Gearman::Protocol.decode_response(response).first
+  end
+
+  def test_decodes_work_warning
+    data     = "I warn you, dude"
+    response = "\0RES" + [Gearman::Protocol::COMMANDS_NUMERIC[:work_warning], data.size].pack("NN") + [handle, data].join("\0")
+    assert_equal [:work_warning, handle, data], Gearman::Protocol.decode_response(response).first
+  end
+
+  def test_decodes_work_data
+    data     = "foo"
+    response = "\0RES" + [Gearman::Protocol::COMMANDS_NUMERIC[:work_data], data.size].pack("NN") + [handle, data].join("\0")
+    assert_equal [:work_data, handle, data], Gearman::Protocol.decode_response(response).first
+  end
+
+  def test_decodes_error
+    data     = "error"
+    response = "\0RES" + [Gearman::Protocol::COMMANDS_NUMERIC[:error], data.size].pack("NN") + [handle, data].join("\0")
+    assert_equal [:error, handle, data], Gearman::Protocol.decode_response(response).first
+  end
+
+  def test_decodes_job_assign
+    function  = "foo_function"
+    arguments = "arguments"
+
+    payload  = [handle, function, arguments].join("\0")
+    response = "\0RES" + [Gearman::Protocol::COMMANDS_NUMERIC[:job_assign], payload.size].pack("NN") + payload
+    assert_equal [:job_assign, handle, function, arguments], Gearman::Protocol.decode_response(response).first
+  end
+
+  def test_decodes_job_assign_uniq
+    function  = "foo_function"
+    arguments = "arguments"
+    unique_id = "123-657"
+
+    payload  = [handle, function, unique_id, arguments].join("\0")
+    response = "\0RES" + [Gearman::Protocol::COMMANDS_NUMERIC[:job_assign_uniq], payload.size].pack("NN") + payload
+    assert_equal [:job_assign_uniq, handle, function, arguments, unique_id], Gearman::Protocol.decode_response(response).first
+  end
+
+  def test_decodes_work_fail
+    response = "\0RES" + [Gearman::Protocol::COMMANDS_NUMERIC[:work_fail], 0].pack("NN") + handle
+    assert_equal [:work_fail, handle], Gearman::Protocol.decode_response(response).first
+  end
+
+  def test_decodes_job_created
+    response = "\0RES" + [Gearman::Protocol::COMMANDS_NUMERIC[:job_created], 0].pack("NN") + handle
+    assert_equal [:job_created, handle], Gearman::Protocol.decode_response(response).first
+  end
+
+  def test_decodes_no_job
+    response = "\0RES" + [Gearman::Protocol::COMMANDS_NUMERIC[:no_job], 0].pack("NN") + handle
+    assert_equal [:no_job, handle], Gearman::Protocol.decode_response(response).first
+  end
+
+  def test_decodes_noop
+    response = "\0RES" + [Gearman::Protocol::COMMANDS_NUMERIC[:noop], 0].pack("NN") + handle
+    assert_equal [:noop, handle], Gearman::Protocol.decode_response(response).first
+  end
+
+  def test_decodes_work_status
+    numerator   = "1"
+    denominator = "5"
+
+    payload  = [handle, numerator, denominator].join("\0")
+    response = "\0RES" + [Gearman::Protocol::COMMANDS_NUMERIC[:work_status], payload.size].pack("NN") + payload
+
+    assert_equal [:work_status, handle, numerator, denominator], Gearman::Protocol.decode_response(response).first
+  end
+
+  def test_decodes_status_res
+    known       = "1"
+    running     = "0"
+    numerator   = "1"
+    denominator = "4"
+
+    payload = [handle, known, running, numerator, denominator].join("\0")
+    response = "\0RES" + [Gearman::Protocol::COMMANDS_NUMERIC[:status_res], payload.size].pack("NN") + payload
+
+    assert_equal [:status_res, handle, known, running, numerator, denominator], Gearman::Protocol.decode_response(response).first
+  end
+
+  def test_raises_on_invalid_command
+    response = "\0RES" + [6969, 0].pack("NN")
+    assert_raises(Gearman::ProtocolError) { Gearman::Protocol.decode_response(response) }
+    assert_raises(Gearman::ProtocolError) { Gearman::Protocol.response_packet(*[6969, handle, ''])}
+  end
+
+private
+  def handle
+    "H:wrk.acme:1"
+  end
+end