gearman-ruby 2.0.0

data/lib/gearman/server.rb

@@ -0,0 +1,94 @@
+#!/usr/bin/env ruby
+
+require 'socket'
+require 'gearman'
+
+module Gearman
+
+# = Server
+#
+# == Description
+# A client for managing Gearman job servers.
+class Server
+  ##
+  # Create a new client.
+  #
+  # @param job_servers  "host:port"; either a single server or an array
+  # @param prefix       function name prefix (namespace)
+  def initialize(hostport)
+    @hostport = hostport  # "host:port"
+  end
+  attr_reader :hostport
+
+  ##
+  # Get a socket for a job server.
+  #
+  # @param hostport  job server "host:port"
+  # @return          a Socket
+  def socket(num_retries=3)
+    return @socket if @socket
+    num_retries.times do
+      begin
+        sock = TCPSocket.new(*hostport.split(':'))
+      rescue Exception
+      else
+        return @socket = sock
+      end
+    end
+    raise RuntimeError, "Unable to connect to job server #{hostport}"
+  end
+
+  ##
+  # Sends a command to the server.
+  #
+  # @return a response string
+  def send_command(name)
+    response = ''
+    socket.puts(name)
+    while true do
+      if buf = socket.recv_nonblock(65536) rescue nil
+        response << buf
+        return response if response =~ /\n.\n$/
+      end
+    end
+  end
+
+  ##
+  # Returns results of a 'status' command.
+  #
+  # @return a hash of abilities with queued, active and workers keys.
+  def status
+    status = {}
+    if response = send_command('status')
+      response.split("\n").each do |line|
+        if line.match /^([A-Za-z_]+)\t([A-Za-z_]+)\t(\d+)\t(\d+)\t(\d+)$/
+          (status[$1] ||= {})[$2] = { :queue => $3, :active => $4, :workers => $5 }
+        end
+      end
+    end
+    status
+  end
+
+  ##
+  # Returns results of a 'workers' command.
+  #
+  # @return an array of worker hashes, containing host, status and functions keys.
+  def workers
+    workers = []
+    if response = send_command('workers')
+      response.split("\n").each do |line|
+        if line.match /^(\d+)\s([a-z0-9\:\.]+)\s([^\s]*)\s:\s([a-z_\s\t]+)$/
+          func_parts = $4.split(' ')
+          functions = []
+          while !func_parts.empty?
+            functions << func_parts.shift << '.' << func_parts.shift
+          end
+          workers << { :host => $2, :status => $3, :functions => functions }
+        end
+      end
+    end
+    workers
+  end
+end
+
+end

data/lib/gearman/task.rb

@@ -0,0 +1,99 @@
+module Gearman
+  class Task
+
+    attr_reader :name, :payload, :retries_done
+    attr_accessor :retries, :priority, :background, :poll_status_interval
+
+    def initialize(name, payload = nil, opts = {})
+      @name       = name.to_s
+      @payload    = payload || ''
+      @priority   = opts.delete(:priority).to_sym rescue nil
+      @background = opts.delete(:background) ? true : false
+
+      @retries_done = 0
+      @retries      = opts.delete(:retries) || 0
+
+      @poll_status_interval = opts.delete(:poll_status_interval)
+      @uniq = opts.has_key?(:uuid) ? opts.delete(:uuid) : `uuidgen`.strip
+    end
+
+    ##
+    # Set a block of code to be executed when this task completes
+    # successfully.  The returned data will be passed to the block.
+    def on_complete(&f)
+      @on_complete = f
+    end
+
+    ##
+    # Set a block of code to be executed when this task fails.
+    def on_fail(&f)
+      @on_fail = f
+    end
+
+    ##
+    # Set a block of code to be executed when this task is retried after
+    # failing.  The number of retries that have been attempted (including the
+    # current one) will be passed to the block.
+    def on_retry(&f)
+      @on_retry = f
+    end
+
+    ##
+    # Set a block of code to be executed when a remote exception is sent by a worker.
+    # The block will receive the message of the exception passed from the worker.
+    # The user can return true for retrying or false to mark it as finished
+    #
+    # NOTE: this is actually deprecated, cf. https://bugs.launchpad.net/gearmand/+bug/405732
+    #
+    def on_exception(&f)
+      @on_exception = f
+    end
+
+    ##
+    # Set a block of code to be executed when we receive a status update for
+    # this task.  The block will receive two arguments, a numerator and
+    # denominator describing the task's status.
+    def on_status(&f)
+      @on_status = f
+    end
+
+    ##
+    # Set a block of code to be executed when we receive a warning from a worker.
+    # It is recommended for workers to send work_warning, followed by work_fail if
+    # an exception occurs on their side. Don't expect this behavior from workers NOT
+    # using this very library ATM, though. (cf. https://bugs.launchpad.net/gearmand/+bug/405732)
+    def on_warning(&f)
+      @on_warning = f
+    end
+
+    ##
+    # Set a block of code to be executed when we receive a (partial) data packet for this task.
+    # The data received will be passed as an argument to the block.
+    def on_data(&f)
+      @on_data = f
+    end
+
+    ##
+    # Record a failure and check whether we should be retried.
+    #
+    # @return  true if we should be resubmitted; false otherwise
+    def should_retry?
+      return false if @retries_done >= @retries
+      @retries_done += 1
+      true
+    end
+
+    def background?
+      background
+    end
+
+    def dispatch(event, *args)
+      callback = instance_variable_get("@#{event}".to_sym)
+      callback.call(*args) if callback
+    end
+
+    def hash
+      @uniq
+    end
+  end
+end

data/lib/gearman/taskset.rb

@@ -0,0 +1,11 @@
+module Gearman
+  class Taskset < ::Array
+
+    def self.create(task_or_taskset)
+      [*task_or_taskset]
+    end
+
+    alias :add :<<
+    alias :add_task :add
+  end
+end

data/lib/gearman/util.rb

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+require 'socket'
+require 'time'
+
+module Gearman
+
+  class ServerDownException < Exception; end
+
+  # = Util
+  #
+  # == Description
+  # Static helper methods and data used by other classes.
+  class Util
+
+    @@debug = false
+
+    ##
+    # Enable or disable debugging output (off by default).
+    #
+    # @param v  print debugging output
+    def Util.debug=(v)
+      @@debug = v
+    end
+
+    ##
+    # Log a message if debugging is enabled.
+    #
+    # @param str  message to log
+    def Util.log(str, force=false)
+      puts "#{Time.now.strftime '%Y-%m-%d %H:%M:%S'} #{str}" if force or @@debug
+    end
+
+    ##
+    # Log a message no matter what.
+    #
+    # @param str  message to log
+    def Util.err(str)
+      log(str, true)
+    end
+
+    def Util.ability_name_with_prefix(prefix,name)
+      "#{prefix}\t#{name}"
+    end
+
+    class << self
+      alias :ability_name_for_perl :ability_name_with_prefix
+    end
+
+  end
+
+end

data/lib/gearman/worker.rb

@@ -0,0 +1,39 @@
+module Gearman
+  class Worker
+
+    attr_reader :abilities
+
+    def initialize(job_servers, opts = {})
+      @reactors = []
+      @abilities = {}
+
+      @job_servers = Array[*job_servers]
+
+      @opts = opts
+    end
+
+    def add_ability(name, timeout = nil, &f)
+      remove_ability(name) if @abilities.has_key?(name)
+      @abilities[name] = { :callback => f, :timeout => timeout }
+    end
+
+    def remove_ability(name)
+      @abilities.delete(name)
+    end
+
+    def has_ability?(name)
+      @abilities.has_key?(name)
+    end
+
+    def work
+      EM.run do
+        @job_servers.each do |hostport|
+          host, port = hostport.split(":")
+          opts = { :abilities => @abilities }.merge(@opts)
+          Gearman::Evented::WorkerReactor.connect(host, port, opts)
+        end
+      end
+    end
+
+  end
+end

data/test/basic_integration_test.rb

@@ -0,0 +1,121 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class BasicIntegrationTest < Test::Unit::TestCase
+
+  def setup
+    start_gearmand
+    @client = Gearman::Client.new("localhost:4730")
+    @worker = Gearman::Worker.new("localhost:4730")
+    Thread.new { EM.run } unless EM.reactor_running?
+  end
+
+  def teardown
+    stop_gearmand
+  end
+
+  def test_ping_job
+    response = nil
+
+    @worker.add_ability("pingpong") {|data, job| "pong" }
+    @worker.work
+
+    task = Gearman::Task.new("pingpong", "ping")
+
+    task.on_complete {|res| response = res }
+    @client.run task
+
+    assert_equal "pong", response
+  end
+
+  def test_exception_in_worker
+    warning_given = nil
+    failed = false
+
+    task = Gearman::Task.new("crash", "doesntmatter")
+
+    @worker.add_ability("crash") {|data, job| raise Exception.new("BOOM!") }
+    @worker.work
+
+    task.on_warning {|warning| warning_given = warning }
+    task.on_fail { failed = true}
+    @client.run task
+
+    assert_not_nil warning_given
+    assert_equal true, failed
+    assert_equal 0, task.retries_done
+  end
+
+  def test_should_be_able_to_retry_on_worker_exception
+    retry_callback_called = false
+    fail_count = 0
+
+    # Gearman::Util.debug = true
+    task = Gearman::Task.new("crash", "doesntmatter", :retries => 1)
+
+    @worker.add_ability("crash") {|data, job| raise Exception.new("BOOM!") }
+    @worker.work
+
+    task.on_retry {|i| retry_callback_called = true }
+    task.on_fail { fail_count += 1 }
+    @client.run task
+
+    assert_equal true, retry_callback_called
+    assert_equal 1, task.retries_done
+    assert_equal 1, fail_count
+  end
+
+  def test_chunked_response
+    chunks_received = 0
+
+    @worker.add_ability("chunked") do |data, job|
+      5.times {|i| job.send_partial("chunk #{i}") }
+    end
+    @worker.work
+
+    task = Gearman::Task.new("chunked")
+    task.on_data do |data|
+      assert_match /^chunk \d/, data
+      chunks_received += 1
+    end
+    @client.run task
+
+    assert_equal 5, chunks_received
+  end
+
+  def test_background
+    status_received = false
+
+    @worker.add_ability("fireandforget") {|data, job| "this goes to /dev/null" }
+    @worker.work
+
+    task = Gearman::Task.new('fireandforget', 'background', :background => true, :poll_status_interval => 0.1)
+    task.on_complete {|d| flunk "on_complete should never be called for a background job!" }
+    task.on_status {|d| status_received = true }
+    @client.run task
+
+    assert_equal true, status_received
+  end
+
+  def test_non_blocking_run
+    @worker.add_ability("foo") {|data, job| "foo: #{data}" }
+    @worker.work
+
+    task1_complete = false
+    task2_complete = false
+
+    task1 = Gearman::Task.new("foo", 1)
+    task1.on_complete {|d| task1_complete = true}
+    @client.run task1, nil, true
+
+    assert_equal false, task1_complete
+
+    task2 = Gearman::Task.new("foo", 2)
+    task2.on_complete {|d| task2_complete = true}
+    @client.run task2, nil, true
+
+    Thread.new { sleep 0.1 }.join
+
+    assert_equal true, task1_complete
+    assert_equal true, task2_complete
+  end
+end

data/test/crash_test.rb

@@ -0,0 +1,69 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+# Gearman::Util.debug = true
+class CrashTest < Test::Unit::TestCase
+
+  def setup
+    teardown_gearmands
+    Thread.new { EM.run } unless EM.reactor_running?
+  end
+
+  def teardown
+    teardown_gearmands
+  end
+
+  def test_worker_should_reconnect_if_gearmand_goes_away
+    start_gearmand
+    worker = Gearman::Worker.new("localhost:4730", :reconnect_sec => 1)
+    worker.add_ability("foo") {|data, job| "noop!" }
+
+    response = nil
+
+    worker.work
+    stop_gearmand
+    start_gearmand
+
+    task = Gearman::Task.new("foo", "ping")
+    task.on_complete {|res| response = res }
+    Gearman::Client.new("localhost:4730").run task
+
+    assert_equal "noop!", response
+    stop_gearmand
+  end
+
+  def test_client_and_worker_should_use_failover_gearmand_if_primary_is_not_available
+    start_gearmand 4731
+
+    worker = Gearman::Worker.new(["localhost:4730", "localhost:4731"], :reconnect_sec => 1)
+    worker.add_ability("foo") {|data, job| "noop!" }
+
+    response = nil
+    worker.work
+    task = Gearman::Task.new("foo", "ping")
+    task.on_complete {|res| response = res }
+    Gearman::Client.new(["localhost:4730", "localhost:4731"]).run task
+
+    assert_equal "noop!", response
+    stop_gearmand 4731
+  end
+
+  def test_client_and_worker_should_switch_to_failover_gearmand_if_primary_goes_down
+    start_gearmand 4730
+    start_gearmand 4731
+
+    worker = Gearman::Worker.new(["localhost:4730", "localhost:4731"], :reconnect_sec => 1)
+    worker.add_ability("foo") {|data, job| "noop!" }
+
+    response = nil
+    worker.work
+
+    stop_gearmand 4730
+
+    task = Gearman::Task.new("foo", "ping")
+    task.on_complete {|res| response = res }
+    Gearman::Client.new(["localhost:4730", "localhost:4731"]).run task
+
+    assert_equal "noop!", response
+    stop_gearmand 4731
+  end
+end