evented-gearman-ruby 1.0.0

data/examples/client_background.rb

@@ -0,0 +1,14 @@
+require 'rubygems'
+#require 'gearman'
+require '../lib/gearman'
+Gearman::Util.debug = true
+
+servers = ['localhost:4730', 'localhost:4731']
+
+client = Gearman::Client.new(servers)
+
+task = Gearman::Task.new('sleep', 20, :background => true, :poll_status_interval => 1)
+task.on_complete {|d| puts d } #never called
+task.on_status {|d| puts "Status: #{d}"}
+
+client.run(task)

data/examples/client_data.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+#require 'gearman'
+require '../lib/gearman'
+Gearman::Util.debug = true
+
+servers = ['localhost:4730']
+
+client = Gearman::Client.new(servers)
+taskset = Gearman::Taskset.new
+
+task = Gearman::Task.new('chunked_transfer')
+task.on_data {|d| puts d }
+task.on_complete {|d| puts d }
+
+taskset << task
+client.run taskset

data/examples/client_exception.rb

@@ -0,0 +1,17 @@
+require 'rubygems'
+require '../lib/gearman'
+Gearman::Util.debug = true
+
+servers = ['localhost:4730']
+
+client = Gearman::Client.new(servers)
+
+task = Gearman::Task.new('fail_with_exception', "void")
+task.retries = 2
+task.on_complete {|d| puts d }
+task.on_exception {|ex| puts "This should never be called" }
+task.on_warning {|warning| puts "WARNING: #{warning}" }
+task.on_retry { puts "PRE-RETRY HOOK: retry no. #{task.retries_done}" }
+task.on_fail { puts "TASK FAILED, GIVING UP" }
+
+client.run task

data/examples/client_prefix.rb

@@ -0,0 +1,15 @@
+require 'rubygems'
+#require 'gearman'
+require '../lib/gearman'
+Gearman::Util.debug = true
+
+servers = ['localhost:4730', 'localhost:4731']
+
+ability_name_with_prefix = Gearman::Util.ability_name_with_prefix("test","sleep")
+
+client = Gearman::Client.new(servers)
+
+task = Gearman::Task.new(ability_name_with_prefix, 20)
+task.on_complete {|d| puts d }
+
+client.run task

data/examples/evented_client.rb

@@ -0,0 +1,23 @@
+require 'rubygems'
+require "ruby-debug"
+require '../lib/gearman'
+
+Gearman::Util.debug = true
+
+servers = ['localhost:4730', 'localhost:4731']
+client = Gearman::Client.new(servers)
+
+taskset = Gearman::Taskset.new
+
+task = Gearman::Task.new('sleep', 2)
+task.on_status {|numerator, denominator| puts "TASK 1: Completed #{numerator} of #{denominator}"}
+task.on_complete {|d| puts "TASK 1: #{d}" }
+taskset << task
+
+task = Gearman::Task.new('sleep', 15, :poll_status_interval => 2, :uuid => nil)
+task.on_status {|numerator, denominator| puts "TASK 2: Completed #{numerator} of #{denominator}"}
+task.on_data {|data| puts "TASK 2 DATA: #{data}" }
+task.on_complete {|d| puts "TASK 2: #{d}" }
+taskset << task
+
+client.run(taskset)

data/examples/evented_worker.rb

@@ -0,0 +1,26 @@
+require 'rubygems'
+require '../lib/gearman'
+
+Gearman::Util.debug = true
+
+servers = ['localhost:4730', 'localhost:4731']
+w = Gearman::Worker.new(servers)
+
+# Add a handler for a "sleep" function that takes a single argument, the
+# number of seconds to sleep before reporting success.
+w.add_ability('sleep') do |data, job|
+ seconds = data
+ job.report_warning("this is a warning you can safely ignore")
+
+ (1..seconds.to_i).each do |i|
+   sleep 1
+   print i
+   # Report our progress to the job server every second.
+   job.report_status(i, seconds)
+   job.send_partial(".")
+ end
+
+ # Report success.
+ "SUCCESS"
+end
+w.work

data/examples/gearman_environment.sh

@@ -0,0 +1,25 @@
+#!/bin/bash  
+
+# Start Gearmand
+echo ' + Starting Gearmand'
+gearmand --daemon --pidfile=/tmp/gearmand.pid
+
+# Start the client and the worker(s)
+echo ' + Starting calculus_worker.rb'
+ruby calculus_worker.rb & 
+
+sleep 3
+
+echo ' + Starting calculus_client.rb'
+ruby calculus_client.rb 
+
+echo ' +++ Example finished +++ '
+
+# Stop Gearmand
+echo ' - Stopping Gearmand'
+kill -9 `cat /tmp/gearmand.pid`
+
+# Stop the workers
+echo ' - Stopping calculus_worker.rb'
+kill -9 `ps ax|grep calculus_worker|grep ruby|awk -F' ' '{print $1}'`
+

data/examples/scale_image.rb

@@ -0,0 +1,30 @@
+#!/usr/bin/ruby
+
+$: << '../lib'
+require 'gearman'
+require 'optparse'
+
+Gearman::Util.debug = true
+servers = 'localhost:7003'
+format = 'PNG'
+width, height = 100, 100
+
+opts = OptionParser.new
+opts.banner = "Usage: #{$0} [options] <input> <output>"
+opts.on('-f FORMAT', '--format', 'Scaled image format') { format }
+opts.on('-h HEIGHT', '--height', 'Scaled image height') { height }
+opts.on('-s SERVERS', '--servers',
+  'Servers, comma-separated host:port') { servers }
+opts.on('-w WIDTH', '--width', 'Scaled image width') { width }
+opts.parse!
+
+if ARGV.size != 2
+  $stderr.puts opts.banner
+  exit 1
+end
+
+client = Gearman::Client.new(servers.split(','), 'example')
+arg = [width, height, format, File.read(ARGV[0])].join("\0")
+task = Gearman::Task.new('scale_image', arg)
+task.on_complete {|d| File.new(ARGV[1],'w').write(d) }
+client.run task

data/examples/scale_image_worker.rb

@@ -0,0 +1,34 @@
+#!/usr/bin/ruby
+
+$: << '../lib'
+require 'gearman'
+require 'optparse'
+require 'RMagick'
+
+Gearman::Util.debug = true
+servers = 'localhost:7003'
+
+opts = OptionParser.new
+opts.banner = "Usage: #{$0} [options]"
+opts.on('-s SERVERS', '--servers',
+  'Job servers, comma-separated host:port') { servers }
+opts.parse!
+
+worker = Gearman::Worker.new(servers.split(','), 'example')
+
+worker.add_ability('scale_image') do |data,job|
+  width, height, format, data = data.split("\0", 4)
+  width = width.to_f
+  height = height.to_f
+  image = Magick::Image.from_blob(data)[0]
+  orig_ratio = image.columns.to_f / image.rows
+  new_ratio = width / height
+  w = new_ratio < orig_ratio ? width : orig_ratio / new_ratio * width
+  h = new_ratio > orig_ratio ? height : new_ratio / orig_ratio * height
+  puts "Got #{image.inspect}; resizing to #{w}x#{h} #{format}"
+  image.resize!(w, h)
+  image.format = format
+  image.to_blob
+end
+
+worker.work

data/examples/server.rb

@@ -0,0 +1,15 @@
+require 'rubygems'
+# require 'gearman'
+# require 'gearman/server'
+require '../lib/gearman'
+require '../lib/gearman/server'
+require 'pp'
+
+Gearman::Util.debug = true
+w = Gearman::Server.new('localhost:4730')
+
+loop {
+  pp "Status: ", w.status
+  pp "Workers: ", w.workers
+  sleep 5
+}

data/examples/worker.rb

@@ -0,0 +1,23 @@
+require 'rubygems'
+#require 'gearman'
+require '../lib/gearman'
+
+Gearman::Util.debug = true
+
+servers = ['localhost:4730', 'localhost:4731']
+w = Gearman::Worker.new(servers)
+
+# Add a handler for a "sleep" function that takes a single argument, the
+# number of seconds to sleep before reporting success.
+w.add_ability('sleep') do |data,job|
+ seconds = data
+ (1..seconds.to_i).each do |i|
+   sleep 1
+   print i
+   # Report our progress to the job server every second.
+   job.report_status(i, seconds)
+ end
+ # Report success.
+ true
+end
+w.work

data/examples/worker_data.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+require '../lib/gearman'
+
+Gearman::Util.debug = true
+
+servers = ['localhost:4730']
+worker = Gearman::Worker.new(servers)
+
+worker.add_ability('chunked_transfer') do |data, job|
+  5.times do |i|
+    sleep 1
+    job.send_partial("CHUNK #{i}")
+  end
+  "EOD"
+end
+worker.work

data/examples/worker_exception.rb

@@ -0,0 +1,14 @@
+require 'rubygems'
+require '../lib/gearman'
+
+Gearman::Util.debug = true
+
+servers = ['localhost:4730']
+w = Gearman::Worker.new(servers)
+
+# Add a handler for a "sleep" function that takes a single argument, the
+# number of seconds to sleep before reporting success.
+w.add_ability('fail_with_exception') do |data,job|
+  raise Exception.new("Exception in worker (args: #{data.inspect})")
+end
+w.work

data/examples/worker_prefix.rb

@@ -0,0 +1,25 @@
+require 'rubygems'
+#require 'gearman'
+require '../lib/gearman'
+
+Gearman::Util.debug = true
+
+servers = ['localhost:4730', 'localhost:4731']
+w = Gearman::Worker.new(servers)
+
+ability_name_with_prefix = Gearman::Util.ability_name_with_prefix("test","sleep")
+
+# Add a handler for a "sleep" function that takes a single argument, the
+# number of seconds to sleep before reporting success.
+w.add_ability(ability_name_with_prefix) do |data,job|
+ seconds = data
+ (1..seconds.to_i).each do |i|
+   sleep 1
+   print i
+   # Report our progress to the job server every second.
+   job.report_status(i, seconds)
+ end
+ # Report success.
+ true
+end
+w.work

data/lib/gearman.rb

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'eventmachine'
+
+module Gearman
+
+  require File.dirname(__FILE__) + '/gearman/evented/reactor'
+  require File.dirname(__FILE__) + '/gearman/evented/client'
+  require File.dirname(__FILE__) + '/gearman/evented/worker'
+  require File.dirname(__FILE__) + '/gearman/client'
+  require File.dirname(__FILE__) + '/gearman/task'
+  require File.dirname(__FILE__) + '/gearman/taskset'
+  require File.dirname(__FILE__) + '/gearman/util'
+  require File.dirname(__FILE__) + '/gearman/worker'
+  require File.dirname(__FILE__) + '/gearman/job'
+
+  require File.dirname(__FILE__) + '/gearman/protocol'
+
+
+  class InvalidArgsError < Exception
+  end
+
+  class NetworkError < Exception
+  end
+
+  def log(msg, force = false)
+    Util.log(msg, force)
+  end
+
+end

data/lib/gearman/client.rb

@@ -0,0 +1,80 @@
+module Gearman
+  class Client
+    attr_accessor :uniq, :jobs
+
+    def initialize(job_servers, opts = {})
+      @reactors = []
+      @jobs = {}
+
+      @job_servers = Array[*job_servers]
+
+      @uniq = opts.delete(:uniq)
+      @opts = opts
+    end
+
+    # Run a Task or Taskset
+    def run(taskset, timeout = nil, async = false)
+      timeout ||= 0
+      use_em_stop = EM.reactor_running?
+      EM.run do
+        @taskset ||= Taskset.new
+        @taskset += Taskset.create(taskset)
+
+        job_servers = @job_servers.dup
+        @reactors.each do |reactor|
+          unless reactor.connected?
+            reactor.reconnect true
+            reactor.keep_connected = async
+            reactor.callback { create_job(@taskset.shift, reactor) }
+            job_servers.delete reactor.to_s
+          else
+            create_job(@taskset.shift, reactor)
+          end
+        end
+        job_servers.each do |hostport|
+          host, port = hostport.split(":")
+          reactor = Gearman::Evented::ClientReactor.connect(host, port, @opts)
+          reactor.keep_connected = async
+          reactor.callback { create_job(@taskset.shift, reactor) }
+          @reactors << reactor
+        end
+
+        if timeout > 0
+          if use_em_stop
+            EM.add_timer(timeout) { EM.stop }
+          else
+            sleep timeout
+          end
+        elsif !async
+          Thread.new do
+            loop do
+              sleep 0.1
+              live = 0
+              @reactors.each {|reactor| live += 1 if reactor.connected? }
+              break if live == 0
+            end
+          end.join
+        end
+      end
+    end
+
+    private
+
+      def create_job(task, reactor = nil)
+        return unless task
+        reactor ||= @reactors[rand(@reactors.size)]
+        unless reactor.connected?
+          log "create_job: server #{reactor} not connected"
+          EM.next_tick { create_job(task) }
+          return
+        end
+
+        reactor.submit_job(task) {|handle| create_job(@taskset.shift) }
+      end
+
+      def log(msg)
+        Gearman::Util.log(msg)
+      end
+
+  end
+end

data/lib/gearman/evented/client.rb

@@ -0,0 +1,99 @@
+module Gearman
+  module Evented
+
+    module ClientReactor
+      include Gearman::Evented::Reactor
+
+      def keep_connected
+        @keep_connected ||= (@opts[:keep_connected] || false)
+      end
+
+      def keep_connected=(keep)
+        @keep_connected = keep
+      end
+
+      def connection_completed
+        @cbs_job_created ||= []
+        @pending_jobs    = []
+        @assigned_jobs   = {}
+        @background_jobs = {}
+        super
+      end
+
+      def receive_data(data)
+        packets = Gearman::Protocol.decode_response(data)
+        log "received #{packets.size} packet(s) at once"
+        log "packets: #{packets.inspect}"
+        packets.each do |type, handle, *data|
+          dispatch_packet(type, handle, *data)
+        end
+      end
+
+      def dispatch_packet_callback(&callback)
+        @dispatch_packet_callback = callback
+      end
+
+      def dispatch_packet(type, handle, *data)
+        log "Got #{type.to_s}, #{handle}, #{data.inspect} from #{server}"
+        if type == :job_created
+          job_created(handle)
+          if cb = @cbs_job_created.shift
+            cb.call(handle)
+          end
+        else
+          dispatch(type, handle, data)
+        end
+      end
+
+      def submit_job(task, &cb_job_created)
+        cmd = "submit_job"
+        cmd << "_#{task.priority}" if [ :high, :low ].include?(task.priority)
+        cmd << "_bg" if task.background
+
+        log "#{cmd} #{task.name}, #{task.payload} to #{server}"
+        send cmd.to_sym, [ task.name, task.hash, task.payload ].join("\0")
+        @pending_jobs << task
+        @cbs_job_created << cb_job_created if cb_job_created
+      end
+
+      def job_created(handle)
+        job = @pending_jobs.shift
+        raise ProtocolError, "No job waiting for handle! (#{handle})" unless job
+        EM.add_periodic_timer(job.poll_status_interval) { get_status(handle) } if job.poll_status_interval
+        if job.background
+          @background_jobs[handle] = job
+        else
+          @assigned_jobs[handle] = job
+        end
+      end
+
+      def get_status(handle)
+        send :get_status, handle
+      end
+
+      def dispatch(type, handle, args)
+        return unless type
+        task = @assigned_jobs[handle]
+        task = @background_jobs[handle] unless task
+        raise ProtocolError, "No task by that name: #{handle}" unless task
+
+        if :work_fail == type && task.should_retry?
+          task.dispatch(:on_retry, task.retries_done)
+          @assigned_jobs.delete(handle)
+          submit_job(task)
+          return
+        end
+
+        if type == :status_res
+          task.dispatch(:on_status, args)
+        else
+          task.dispatch(type.to_s.sub("work", "on"), *args)
+        end
+
+        @assigned_jobs.delete(handle) if [:work_complete, :work_fail].include?(type)
+        disconnect if @assigned_jobs.empty? && !keep_connected
+      end
+    end
+
+  end
+end