xing-gearman-ruby 1.2.0 → 1.3.0

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :patch: 0
 :major: 1
-:minor: 2
+:minor: 3

data/examples/client_data.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+#require 'gearman'
+require '../lib/gearman'
+Gearman::Util.debug = true
+
+servers = ['localhost:4730']
+  
+client = Gearman::Client.new(servers)
+taskset = Gearman::TaskSet.new(client)
+
+task = Gearman::Task.new('chunked_transfer')
+task.on_data {|d| puts d }
+task.on_complete {|d| puts d }
+
+taskset.add_task(task)
+taskset.wait(100)

data/examples/worker_data.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+require '../lib/gearman'
+
+Gearman::Util.debug = true
+
+servers = ['localhost:4730']
+worker = Gearman::Worker.new(servers)
+
+worker.add_ability('chunked_transfer') do |data, job|
+  5.times do |i|
+    sleep 1
+    job.send_data("CHUNK #{i}")
+  end
+  "EOD"
+end
+loop { worker.work }

data/gearman-ruby.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{gearman-ruby}
-  s.version = "1.2.0"
+  s.version = "1.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Daniel Erat", "Ladislav Martincik", "Pablo Delgado", "Mauro Pompilio", "Antonio Garrote", "Kim Altintop"]
-  s.date = %q{2009-07-30}
+  s.date = %q{2009-08-04}
   s.description = %q{Library for the Gearman distributed job system}
   s.email = %q{ladislav.martincik@xing.com}
   s.extra_rdoc_files = [
@@ -37,6 +37,7 @@ Gem::Specification.new do |s|
      "examples/client.php",
      "examples/client.rb",
      "examples/client_background.rb",
+     "examples/client_data.rb",
      "examples/client_exception.rb",
      "examples/client_prefix.rb",
      "examples/gearman_environment.sh",
@@ -45,6 +46,7 @@ Gem::Specification.new do |s|
      "examples/server.rb",
      "examples/worker.php",
      "examples/worker.rb",
+     "examples/worker_data.rb",
      "examples/worker_exception.rb",
      "examples/worker_prefix.rb",
      "gearman-ruby.gemspec",
@@ -77,12 +79,14 @@ Gem::Specification.new do |s|
      "examples/calculus_worker.rb",
      "examples/client.rb",
      "examples/client_background.rb",
+     "examples/client_data.rb",
      "examples/client_exception.rb",
      "examples/client_prefix.rb",
      "examples/scale_image.rb",
      "examples/scale_image_worker.rb",
      "examples/server.rb",
      "examples/worker.rb",
+     "examples/worker_data.rb",
      "examples/worker_exception.rb",
      "examples/worker_prefix.rb"
   ]

data/lib/gearman/task.rb

@@ -17,7 +17,7 @@ class Task
   def initialize(func, arg='', opts={})
     @func = func.to_s
     @arg = arg or ''  # TODO: use something more ref-like?
-    %w{on_complete on_fail on_retry on_exception on_status on_warning 
+    %w{on_complete on_fail on_retry on_exception on_status on_warning on_data
        uniq retry_count priority background}.map {|s| s.to_sym }.each do |k|
       instance_variable_set "@#{k}", opts[k]
       opts.delete k
@@ -91,6 +91,13 @@ class Task
     @on_warning = f
   end
 
+  ##
+  # Set a block of code to be executed when we receive a (partial) data packet for this task.
+  # The data received will be passed as an argument to the block.
+  def on_data(&f)
+    @on_data = f
+  end
+
   ##
   # Handle completion of the task.
   #
@@ -129,7 +136,7 @@ class Task
     @on_status.call(numerator, denominator) if @on_status
     self
   end
-  
+
   ##
   # Handle a warning.
   #
@@ -138,6 +145,13 @@ class Task
     self
   end
 
+  ##
+  # Handle (partial) data
+  def handle_data(data)
+    @on_data.call(data) if @on_data
+    self
+  end
+
   ##
   # Return a hash that we can use to execute identical tasks on the same
   # job server.

data/lib/gearman/taskset.rb

@@ -113,15 +113,8 @@ class TaskSet
   # @param data      data returned in packet from server
   def handle_work_complete(hostport, data)
     handle, data = data.split("\0", 2)
-    Util.log "Got work_complete with handle #{handle} and " +
-      "#{data ? data.size : '0'} byte(s) of data from #{hostport}"
-    js_handle = Util.handle_to_str(hostport, handle)
-    tasks = @tasks_in_progress.delete(js_handle)
-    if not tasks
-      raise ProtocolError, "Got unexpected work_complete with handle " +
-        "#{handle} from #{hostport} (no task by that name)"
-    end
-    tasks.each do |t|
+    Util.log "Got work_complete with handle #{handle} and #{data ? data.size : '0'} byte(s) of data from #{hostport}"
+    tasks_in_progress(hostport, handle, true).each do |t|
       t.handle_completion(data)
       @finished_tasks << t
     end
@@ -137,13 +130,7 @@ class TaskSet
   def handle_work_exception(hostport, data)
     handle, exception = data.split("\0", 2)
     Util.log "Got work_exception with handle #{handle} from #{hostport}: '#{exception}'"
-    js_handle = Util.handle_to_str(hostport, handle)
-    tasks = @tasks_in_progress[js_handle]
-    if not tasks
-      raise ProtocolError, "Got unexpected work_exception with handle " +
-        "#{handle} from #{hostport} (no task by that name)"
-    end
-    tasks.each {|t| t.handle_exception(exception) }
+    tasks_in_progress(hostport, handle).each {|t| t.handle_exception(exception) }
   end
   private :handle_work_exception
 
@@ -154,13 +141,7 @@ class TaskSet
   # @param data      data returned in packet from server
   def handle_work_fail(hostport, data)
     Util.log "Got work_fail with handle #{data} from #{hostport}"
-    js_handle = Util.handle_to_str(hostport, data)
-    tasks = @tasks_in_progress.delete(js_handle)
-    if not tasks
-      raise ProtocolError, "Got unexpected work_fail with handle " +
-        "#{data} from #{hostport} (no task by that name)"
-    end
-    tasks.each do |t|
+    tasks_in_progress(hostport, data, true).each do |t|
       if t.handle_failure
         add_task_internal(t, false)
       else
@@ -177,15 +158,8 @@ class TaskSet
   # @param data      data returned in packet from server
   def handle_work_status(hostport, data)
     handle, num, den = data.split("\0", 3)
-    Util.log "Got work_status with handle #{handle} from #{hostport}: " +
-      "#{num}/#{den}"
-    js_handle = Util.handle_to_str(hostport, handle)
-    tasks = @tasks_in_progress[js_handle]
-    if not tasks
-      raise ProtocolError, "Got unexpected work_status with handle " +
-        "#{handle} from #{hostport} (no task by that name)"
-    end
-    tasks.each {|t| t.handle_status(num, den) }
+    Util.log "Got work_status with handle #{handle} from #{hostport}: #{num}/#{den}"
+    tasks_in_progress(hostport, handle).each {|t| t.handle_status(num, den) }
   end
   private :handle_work_status
 
@@ -197,15 +171,27 @@ class TaskSet
   def handle_work_warning(hostport, data)
     handle, message = data.split("\0", 2)
     Util.log "Got work_warning with handle #{handle} from #{hostport}: '#{message}'"
+    tasks_in_progress(hostport, handle).each {|t| t.handle_warning(message) }
+  end
+  private :handle_work_warning
+
+  ##
+  # Handle a 'work_data' response from a job server
+  #
+  # @param hostport  "host:port" of a job server
+  # @param data       data returned in packet from server
+  def handle_work_data(hostport, data)
+    handle, data = data.split("\0", 2)
+    Util.log "Got work_data with handle #{handle} and #{data ? data.size : '0'} byte(s) of data from #{hostport}"
+
     js_handle = Util.handle_to_str(hostport, handle)
     tasks = @tasks_in_progress[js_handle]
     if not tasks
-      raise ProtocolError, "Got unexpected work_warning with handle " +
-        "#{handle} from #{hostport} (no task by that name)"
+      raise ProtocolError, "Got unexpected work_data with handle #{handle} from #{hostport} (no task by that name)"
     end
-    tasks.each {|t| t.handle_warning(message) }
+    tasks.each {|t| t.handle_data(data) }
   end
-  private :handle_work_warning
+  private :handle_work_data
 
   ##
   # Read and process a packet from a socket.
@@ -218,19 +204,16 @@ class TaskSet
         sock.inspect
     end
     type, data = Util.read_response(sock, timeout)
-    case type
-    when :job_created
-      handle_job_created(hostport, data)
-    when :work_complete
-      handle_work_complete(hostport, data)
-    when :work_fail
-      handle_work_fail(hostport, data)
-    when :work_status
-      handle_work_status(hostport, data)
-    when :work_exception
-      handle_work_exception(hostport, data)
-    when :work_warning
-      handle_work_warning(hostport, data)
+    known_types = [ :job_created,
+                    :work_complete,
+                    :work_fail,
+                    :work_status,
+                    :work_exception,
+                    :work_warning,
+                    :work_data ]
+
+    if known_types.include?(type)
+      send("handle_#{type}".to_sym, hostport, data)
     else
       Util.log "Got #{type.to_s} from #{hostport}"
     end
@@ -278,6 +261,16 @@ class TaskSet
     end
     true
   end
+  
+  private
+    def tasks_in_progress(hostport, handle, remove_task = false)
+      js_handle = Util.handle_to_str(hostport, handle)
+      tasks = remove_task ? @tasks_in_progress.delete(js_handle) : @tasks_in_progress[js_handle]
+      if not tasks
+        raise ProtocolError, "Got unexpected work_data with handle #{handle} from #{hostport} (no task by that name)"
+      end
+      tasks
+    end
 end
 
 end

data/lib/gearman/util.rb

@@ -15,43 +15,42 @@ class Util
   # Map from Integer representations of commands used in the network
   # protocol to more-convenient symbols.
   COMMANDS = {
-    1  => :can_do,          # W->J: FUNC
-    23 => :can_do_timeout,  # W->J: FUNC[0]TIMEOUT
-    2  => :cant_do,         # W->J: FUNC
-    3  => :reset_abilities, # W->J: --
-    22 => :set_client_id,   # W->J: [RANDOM_STRING_NO_WHITESPACE]
-    4  => :pre_sleep,       # W->J: --
-
-    6  => :noop,            # J->W: --
-    7  => :submit_job,      # C->J: FUNC[0]UNIQ[0]ARGS
-    33 => :submit_job_low,     # C->J: FUNC[0]UNIQ[0]ARGS
-    34 => :submit_job_low_bg,  # C->J: FUNC[0]UNIQ[0]ARGS
-    21 => :submit_job_high,    # C->J: FUNC[0]UNIQ[0]ARGS
-    32 => :submit_job_high_bg, # C->J: FUNC[0]UNIQ[0]ARGS
-    18 => :submit_job_bg,      # C->J: FUNC[0]UNIQ[0]ARGS
-
-    8  => :job_created,     # J->C: HANDLE
-    9  => :grab_job,        # W->J: --
-    10 => :no_job,          # J->W: --
-    11 => :job_assign,      # J->W: HANDLE[0]FUNC[0]ARG
-
-    12 => :work_status,     # W->J/C: HANDLE[0]NUMERATOR[0]DENOMINATOR
-    13 => :work_complete,   # W->J/C: HANDLE[0]RES
-    14 => :work_fail,       # W->J/C: HANDLE
-    29 => :work_warning,    # W->J/C: HANDLE[0]MSG
-
-    25 => :work_exception,  # W->J: HANDLE[0]ARG
-    26 => :option_req,      # C->J: TEXT
-    27 => :option_res,      # J->C: TEXT
-
-    15 => :get_status,      # C->J: HANDLE
-    20 => :status_res,      # C->J: HANDLE[0]KNOWN[0]RUNNING[0]NUM[0]DENOM
-
-    25 => :work_exception,  # W->J/C: HANDLE[0]ARG
-    16 => :echo_req,        # ?->J: TEXT
-    17 => :echo_res,        # J->?: TEXT
-
-    19 => :error,           # J->?: ERRCODE[0]ERR_TEXT
+    1  => :can_do,               # W->J: FUNC
+    2  => :cant_do,              # W->J: FUNC
+    3  => :reset_abilities,      # W->J: --
+    4  => :pre_sleep,            # W->J: --
+    #5 =>  (unused),             # -      -
+    6  => :noop,                 # J->W: --
+    7  => :submit_job,           # C->J: FUNC[0]UNIQ[0]ARGS
+    8  => :job_created,          # J->C: HANDLE
+    9  => :grab_job,             # W->J: --
+    10 => :no_job,               # J->W: --
+    11 => :job_assign,           # J->W: HANDLE[0]FUNC[0]ARG
+    12 => :work_status,          # W->J/C: HANDLE[0]NUMERATOR[0]DENOMINATOR
+    13 => :work_complete,        # W->J/C: HANDLE[0]RES
+    14 => :work_fail,            # W->J/C: HANDLE
+    15 => :get_status,           # C->J: HANDLE
+    16 => :echo_req,             # ?->J: TEXT
+    17 => :echo_res,             # J->?: TEXT
+    18 => :submit_job_bg,        # C->J: FUNC[0]UNIQ[0]ARGS
+    19 => :error,                # J->?: ERRCODE[0]ERR_TEXT
+    20 => :status_res,           # C->J: HANDLE[0]KNOWN[0]RUNNING[0]NUM[0]DENOM
+    21 => :submit_job_high,      # C->J: FUNC[0]UNIQ[0]ARGS
+    22 => :set_client_id,        # W->J: [RANDOM_STRING_NO_WHITESPACE]
+    23 => :can_do_timeout,       # W->J: FUNC[0]TIMEOUT
+    24 => :all_yours,            # REQ    Worker
+    25 => :work_exception,       # W->J: HANDLE[0]ARG
+    26 => :option_req,           # C->J: TEXT
+    27 => :option_res,           # J->C: TEXT
+    28 => :work_data,            # REQ    Worker
+    29 => :work_warning,         # W->J/C: HANDLE[0]MSG
+    30 => :grab_job_uniq,        # REQ    Worker
+    31 => :job_assign_uniq,      # RES    Worker
+    32 => :submit_job_high_bg,   # C->J: FUNC[0]UNIQ[0]ARGS
+    33 => :submit_job_low,       # C->J: FUNC[0]UNIQ[0]ARGS
+    34 => :submit_job_low_bg,    # C->J: FUNC[0]UNIQ[0]ARGS
+    35 => :submit_job_sched,     # REQ    Client
+    36 => :submit_job_epoch      # REQ    Client
   }
 
   # Map e.g. 'can_do' => 1

data/lib/gearman/worker.rb

@@ -79,6 +79,22 @@ class Worker
       Util.send_request(@socket, req)
       self
     end
+
+    ##
+    # Send data before job completes
+    def send_data(data)
+      req = Util.pack_request(:work_data, "#{@handle}\0#{data}")
+      Util.send_request(@socket, req)
+      self
+    end
+
+    ##
+    # Send a warning explicitly
+    def report_warning(warning)
+      req = Util.pack_request(:work_warning, "#{@handle}\0#{warning}")
+      Util.send_request(@socket, req)
+      self
+    end
   end
 
   ##

data/test/mock_client_test.rb

@@ -218,7 +218,7 @@ class TestClient < Test::Unit::TestCase
   def test_callbacks
     server = FakeJobServer.new(self)
     client, task, taskset, sock = nil
-    failed, retries, num, den = nil
+    failed, retries, num, den, warning = nil
 
     s = TestScript.new
     c = TestScript.new
@@ -228,18 +228,20 @@ class TestClient < Test::Unit::TestCase
 
     c.exec { client = Gearman::Client.new("localhost:#{server.port}") }
 
-    task = Gearman::Task.new('foo', 'bar',
-      { :retry_count => 2 })
+    task = Gearman::Task.new('foo', 'bar', { :retry_count => 3 })
     task.on_fail { failed = true }
     task.on_retry {|r| retries = r }
     task.on_status {|n,d| num = n.to_i; den = d.to_i }
+    task.on_warning {|msg| warning = msg }
+    received_data = ""
+    task.on_data {|data| received_data << data }
 
     c.exec { taskset = Gearman::TaskSet.new(client) }
     c.exec { taskset.add_task(task) }
     s.exec { sock = server.expect_connection }
     s.wait
 
-    # Send three failures back to the client.
+    # Send 4 failures back to the client.
     c.exec { taskset.wait }
     s.exec { server.expect_request(sock, :submit_job, "foo\000\000bar") }
     s.exec { server.send_response(sock, :job_created, 'a') }
@@ -251,13 +253,20 @@ class TestClient < Test::Unit::TestCase
     s.exec { server.send_response(sock, :job_created, 'c') }
     s.exec { server.send_response(sock, :work_status, "c\0001\0002") }
     s.exec { server.send_response(sock, :work_fail, 'c') }
+    s.exec { server.send_response(sock, :job_created, 'd') }
+    s.exec { server.send_response(sock, :work_data, "d\000data chunk 1") }
+    s.exec { server.send_response(sock, :work_data, "d\000data chunk 2") }
+    s.exec { server.send_response(sock, :work_warning, "d\000warning") }
+    s.exec { server.send_response(sock, :work_fail, 'd') }
     c.wait
     s.wait
 
     assert_equal(true, failed)
-    assert_equal(2, retries)
+    assert_equal(3, retries)
     assert_equal(1, num)
     assert_equal(2, den)
+    assert_equal("data chunk 1data chunk 2", received_data)
+    assert_equal("warning", warning)
   end
 
   def test_failure
@@ -582,4 +591,39 @@ class TestClient < Test::Unit::TestCase
 
     assert_equal(false, res)
   end
+  
+  ##
+  # Tests partial data responses (work_data)
+  def test_chunked_data
+    server = FakeJobServer.new(self)
+    client, sock, task, taskset, res = nil
+
+    s = TestScript.new
+    c = TestScript.new
+
+    server_thread = Thread.new { s.loop_forever }.run
+    client_thread = Thread.new { c.loop_forever }.run
+
+    c.exec { client = Gearman::Client.new("localhost:#{server.port}") }
+    task = Gearman::Task.new('foo', 'bar', { :retry_count => 3 })
+    received_data = ""
+    task.on_data {|data| received_data << data }
+    task.on_complete {|data| received_data << data}
+
+    c.exec { taskset = Gearman::TaskSet.new(client) }
+    c.exec { taskset.add_task(task) }
+    s.exec { sock = server.expect_connection }
+    s.wait
+
+    c.exec { taskset.wait }
+    s.exec { server.expect_request(sock, :submit_job, "foo\000\000bar") }
+    s.exec { server.send_response(sock, :job_created, 'a') }
+    s.exec { server.send_response(sock, :work_data, "a\000data chunk 1\n") }
+    s.exec { server.send_response(sock, :work_data, "a\000data chunk 2\n") }
+    s.exec { server.send_response(sock, :work_complete, "a\000data complete") }
+    c.wait
+    s.wait
+
+    assert_equal("data chunk 1\ndata chunk 2\ndata complete", received_data)
+  end
 end

data/test/mock_worker_test.rb

@@ -41,7 +41,14 @@ class TestWorker < Test::Unit::TestCase
     # After it connects, it should send its ID, and it should tell us its
     # abilities when we report them.
     s.exec { @server.expect_request(sock, :set_client_id, 'test') }
-    w.exec { worker.add_ability('echo') {|d,j| j.report_status(1, 1); d } }
+    w.exec do
+      worker.add_ability('echo') do |data, job| 
+        job.report_status(1, 1);
+        part1, part2 = data.split(//, 2)
+        job.send_data(part1) # send partial data first
+        part2
+      end
+    end
     s.exec { @server.expect_request(sock, :can_do, 'echo') }
 
     # It should try to grab a job when we tell it to work.
@@ -59,7 +66,8 @@ class TestWorker < Test::Unit::TestCase
     # When we give it a job, it should do it.
     s.exec { @server.send_response(sock, :job_assign, "a\0echo\0foo") }
     s.exec { @server.expect_request(sock, :work_status, "a\0001\0001") }
-    s.exec { @server.expect_request(sock, :work_complete, "a\0foo") }
+    s.exec { @server.expect_request(sock, :work_data, "a\000f") }
+    s.exec { @server.expect_request(sock, :work_complete, "a\0oo") }
 
     # Test that functions are unregistered correctly.
     w.exec { worker.remove_ability('echo') }
@@ -250,7 +258,10 @@ class TestWorker < Test::Unit::TestCase
 
     # When we give it a job, it should raise an excpetion and notify the server
     s.exec { @server.send_response(sock, :job_assign, "a\0echo\0foo") }
-    s.exec { @server.expect_request(sock, :work_exception, "a\0fooexception") }
+    s.exec do
+      @server.expect_request(sock, :work_warning, "a\0fooexception")
+      @server.expect_request(sock, :work_fail, "a")
+    end
 
     s.wait
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: xing-gearman-ruby
 version: !ruby/object:Gem::Version 
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors: 
 - Daniel Erat
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-07-30 00:00:00 -07:00
+date: 2009-08-04 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -51,6 +51,7 @@ files:
 - examples/client.php
 - examples/client.rb
 - examples/client_background.rb
+- examples/client_data.rb
 - examples/client_exception.rb
 - examples/client_prefix.rb
 - examples/gearman_environment.sh
@@ -59,6 +60,7 @@ files:
 - examples/server.rb
 - examples/worker.php
 - examples/worker.rb
+- examples/worker_data.rb
 - examples/worker_exception.rb
 - examples/worker_prefix.rb
 - gearman-ruby.gemspec
@@ -112,11 +114,13 @@ test_files:
 - examples/calculus_worker.rb
 - examples/client.rb
 - examples/client_background.rb
+- examples/client_data.rb
 - examples/client_exception.rb
 - examples/client_prefix.rb
 - examples/scale_image.rb
 - examples/scale_image_worker.rb
 - examples/server.rb
 - examples/worker.rb
+- examples/worker_data.rb
 - examples/worker_exception.rb
 - examples/worker_prefix.rb