gearman-ruby 2.0.0

data/examples/evented_worker.rb

@@ -0,0 +1,26 @@
+require 'rubygems'
+require '../lib/gearman'
+
+Gearman::Util.debug = true
+
+servers = ['localhost:4730', 'localhost:4731']
+w = Gearman::Worker.new(servers)
+
+# Add a handler for a "sleep" function that takes a single argument, the
+# number of seconds to sleep before reporting success.
+w.add_ability('sleep') do |data, job|
+ seconds = data
+ job.report_warning("this is a warning you can safely ignore")
+
+ (1..seconds.to_i).each do |i|
+   sleep 1
+   print i
+   # Report our progress to the job server every second.
+   job.report_status(i, seconds)
+   job.send_partial(".")
+ end
+
+ # Report success.
+ "SUCCESS"
+end
+w.work

data/examples/gearman_environment.sh

@@ -0,0 +1,25 @@
+#!/bin/bash  
+
+# Start Gearmand
+echo ' + Starting Gearmand'
+gearmand --daemon --pidfile=/tmp/gearmand.pid
+
+# Start the client and the worker(s)
+echo ' + Starting calculus_worker.rb'
+ruby calculus_worker.rb & 
+
+sleep 3
+
+echo ' + Starting calculus_client.rb'
+ruby calculus_client.rb 
+
+echo ' +++ Example finished +++ '
+
+# Stop Gearmand
+echo ' - Stopping Gearmand'
+kill -9 `cat /tmp/gearmand.pid`
+
+# Stop the workers
+echo ' - Stopping calculus_worker.rb'
+kill -9 `ps ax|grep calculus_worker|grep ruby|awk -F' ' '{print $1}'`
+

data/examples/scale_image.rb

@@ -0,0 +1,30 @@
+#!/usr/bin/ruby
+
+$: << '../lib'
+require 'gearman'
+require 'optparse'
+
+Gearman::Util.debug = true
+servers = 'localhost:7003'
+format = 'PNG'
+width, height = 100, 100
+
+opts = OptionParser.new
+opts.banner = "Usage: #{$0} [options] <input> <output>"
+opts.on('-f FORMAT', '--format', 'Scaled image format') { format }
+opts.on('-h HEIGHT', '--height', 'Scaled image height') { height }
+opts.on('-s SERVERS', '--servers',
+  'Servers, comma-separated host:port') { servers }
+opts.on('-w WIDTH', '--width', 'Scaled image width') { width }
+opts.parse!
+
+if ARGV.size != 2
+  $stderr.puts opts.banner
+  exit 1
+end
+
+client = Gearman::Client.new(servers.split(','), 'example')
+arg = [width, height, format, File.read(ARGV[0])].join("\0")
+task = Gearman::Task.new('scale_image', arg)
+task.on_complete {|d| File.new(ARGV[1],'w').write(d) }
+client.run task

data/examples/scale_image_worker.rb

@@ -0,0 +1,34 @@
+#!/usr/bin/ruby
+
+$: << '../lib'
+require 'gearman'
+require 'optparse'
+require 'RMagick'
+
+Gearman::Util.debug = true
+servers = 'localhost:7003'
+
+opts = OptionParser.new
+opts.banner = "Usage: #{$0} [options]"
+opts.on('-s SERVERS', '--servers',
+  'Job servers, comma-separated host:port') { servers }
+opts.parse!
+
+worker = Gearman::Worker.new(servers.split(','), 'example')
+
+worker.add_ability('scale_image') do |data,job|
+  width, height, format, data = data.split("\0", 4)
+  width = width.to_f
+  height = height.to_f
+  image = Magick::Image.from_blob(data)[0]
+  orig_ratio = image.columns.to_f / image.rows
+  new_ratio = width / height
+  w = new_ratio < orig_ratio ? width : orig_ratio / new_ratio * width
+  h = new_ratio > orig_ratio ? height : new_ratio / orig_ratio * height
+  puts "Got #{image.inspect}; resizing to #{w}x#{h} #{format}"
+  image.resize!(w, h)
+  image.format = format
+  image.to_blob
+end
+
+worker.work

data/examples/server.rb

@@ -0,0 +1,15 @@
+require 'rubygems'
+# require 'gearman'
+# require 'gearman/server'
+require '../lib/gearman'
+require '../lib/gearman/server'
+require 'pp'
+
+Gearman::Util.debug = true
+w = Gearman::Server.new('localhost:4730')
+
+loop {
+  pp "Status: ", w.status
+  pp "Workers: ", w.workers
+  sleep 5
+}

data/examples/worker.rb

@@ -0,0 +1,23 @@
+require 'rubygems'
+#require 'gearman'
+require '../lib/gearman'
+
+Gearman::Util.debug = true
+
+servers = ['localhost:4730', 'localhost:4731']
+w = Gearman::Worker.new(servers)
+
+# Add a handler for a "sleep" function that takes a single argument, the
+# number of seconds to sleep before reporting success.
+w.add_ability('sleep') do |data,job|
+ seconds = data
+ (1..seconds.to_i).each do |i|
+   sleep 1
+   print i
+   # Report our progress to the job server every second.
+   job.report_status(i, seconds)
+ end
+ # Report success.
+ true
+end
+w.work

data/examples/worker_data.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+require '../lib/gearman'
+
+Gearman::Util.debug = true
+
+servers = ['localhost:4730']
+worker = Gearman::Worker.new(servers)
+
+worker.add_ability('chunked_transfer') do |data, job|
+  5.times do |i|
+    sleep 1
+    job.send_partial("CHUNK #{i}")
+  end
+  "EOD"
+end
+worker.work

data/examples/worker_echo.rb

@@ -0,0 +1,20 @@
+require 'rubygems'
+require '../lib/gearman'
+# require 'perftools'
+
+quit = false
+# PerfTools::CpuProfiler.start("/tmp/worker_echo_profile") do  
+  servers = ['localhost:4730']
+  worker = Gearman::Worker.new(servers)
+
+  worker.add_ability('echo') do |data, job|
+    data
+  end
+  worker.add_ability('quit') do |data, job|
+    quit = true
+  end
+  loop {
+    break if quit
+    worker.work
+  }
+# end

data/examples/worker_echo_pprof.rb

@@ -0,0 +1,5 @@
+require 'rubygems'
+require 'perftools'
+PerfTools::CpuProfiler.start("/tmp/worker_echo_profile") do
+  require File.dirname(__FILE__) + "/worker_echo.rb"
+end

data/examples/worker_exception.rb

@@ -0,0 +1,14 @@
+require 'rubygems'
+require '../lib/gearman'
+
+Gearman::Util.debug = true
+
+servers = ['localhost:4730']
+w = Gearman::Worker.new(servers)
+
+# Add a handler for a "sleep" function that takes a single argument, the
+# number of seconds to sleep before reporting success.
+w.add_ability('fail_with_exception') do |data,job|
+  raise Exception.new("Exception in worker (args: #{data.inspect})")
+end
+w.work

data/examples/worker_prefix.rb

@@ -0,0 +1,25 @@
+require 'rubygems'
+#require 'gearman'
+require '../lib/gearman'
+
+Gearman::Util.debug = true
+
+servers = ['localhost:4730', 'localhost:4731']
+w = Gearman::Worker.new(servers)
+
+ability_name_with_prefix = Gearman::Util.ability_name_with_prefix("test","sleep")
+
+# Add a handler for a "sleep" function that takes a single argument, the
+# number of seconds to sleep before reporting success.
+w.add_ability(ability_name_with_prefix) do |data,job|
+ seconds = data
+ (1..seconds.to_i).each do |i|
+   sleep 1
+   print i
+   # Report our progress to the job server every second.
+   job.report_status(i, seconds)
+ end
+ # Report success.
+ true
+end
+w.work

data/gearman-ruby.gemspec

@@ -0,0 +1,111 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{gearman-ruby}
+  s.version = "2.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Kim Altintop"]
+  s.date = %q{2009-12-11}
+  s.description = %q{Library for the Gearman distributed job system}
+  s.email = %q{kim.altintop@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README"
+  ]
+  s.files = [
+    ".gitignore",
+     "HOWTO",
+     "LICENSE",
+     "README",
+     "Rakefile",
+     "TODO",
+     "VERSION.yml",
+     "examples/calculus_client.rb",
+     "examples/calculus_worker.rb",
+     "examples/client.rb",
+     "examples/client_background.rb",
+     "examples/client_data.rb",
+     "examples/client_exception.rb",
+     "examples/client_prefix.rb",
+     "examples/evented_client.rb",
+     "examples/evented_worker.rb",
+     "examples/gearman_environment.sh",
+     "examples/scale_image.rb",
+     "examples/scale_image_worker.rb",
+     "examples/server.rb",
+     "examples/worker.rb",
+     "examples/worker_data.rb",
+     "examples/worker_exception.rb",
+     "examples/worker_prefix.rb",
+     "gearman-ruby.gemspec",
+     "lib/gearman.rb",
+     "lib/gearman/client.rb",
+     "lib/gearman/evented/client.rb",
+     "lib/gearman/evented/reactor.rb",
+     "lib/gearman/evented/worker.rb",
+     "lib/gearman/job.rb",
+     "lib/gearman/protocol.rb",
+     "lib/gearman/server.rb",
+     "lib/gearman/task.rb",
+     "lib/gearman/taskset.rb",
+     "lib/gearman/util.rb",
+     "lib/gearman/worker.rb",
+     "test/basic_integration_test.rb",
+     "test/crash_test.rb",
+     "test/job_test.rb",
+     "test/protocol_test.rb",
+     "test/test_helper.rb",
+     "test/util_test.rb",
+     "test/worker_test.rb"
+  ]
+  s.homepage = %q{http://github.com/kim/gearman-ruby}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Library for the Gearman distributed job system}
+  s.test_files = [
+    "test/basic_integration_test.rb",
+     "test/crash_test.rb",
+     "test/job_test.rb",
+     "test/protocol_test.rb",
+     "test/test_helper.rb",
+     "test/util_test.rb",
+     "test/worker_test.rb",
+     "examples/calculus_client.rb",
+     "examples/calculus_worker.rb",
+     "examples/client.rb",
+     "examples/client_background.rb",
+     "examples/client_data.rb",
+     "examples/client_echo.rb",
+     "examples/client_exception.rb",
+     "examples/client_prefix.rb",
+     "examples/evented_client.rb",
+     "examples/evented_worker.rb",
+     "examples/scale_image.rb",
+     "examples/scale_image_worker.rb",
+     "examples/server.rb",
+     "examples/worker.rb",
+     "examples/worker_data.rb",
+     "examples/worker_echo.rb",
+     "examples/worker_echo_pprof.rb",
+     "examples/worker_exception.rb",
+     "examples/worker_prefix.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<eventmachine>, [">= 0.12.8"])
+    else
+      s.add_dependency(%q<eventmachine>, [">= 0.12.8"])
+    end
+  else
+    s.add_dependency(%q<eventmachine>, [">= 0.12.8"])
+  end
+end

data/lib/gearman.rb

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'eventmachine'
+
+module Gearman
+
+  require File.dirname(__FILE__) + '/gearman/evented/reactor'
+  require File.dirname(__FILE__) + '/gearman/evented/client'
+  require File.dirname(__FILE__) + '/gearman/evented/worker'
+  require File.dirname(__FILE__) + '/gearman/client'
+  require File.dirname(__FILE__) + '/gearman/task'
+  require File.dirname(__FILE__) + '/gearman/taskset'
+  require File.dirname(__FILE__) + '/gearman/util'
+  require File.dirname(__FILE__) + '/gearman/worker'
+  require File.dirname(__FILE__) + '/gearman/job'
+
+  require File.dirname(__FILE__) + '/gearman/protocol'
+
+
+  class InvalidArgsError < Exception
+  end
+
+  class NetworkError < Exception
+  end
+
+  def log(msg, force = false)
+    Util.log(msg, force)
+  end
+
+end

data/lib/gearman/client.rb

@@ -0,0 +1,80 @@
+module Gearman
+  class Client
+    attr_accessor :uniq, :jobs
+
+    def initialize(job_servers, opts = {})
+      @reactors = []
+      @jobs = {}
+
+      @job_servers = Array[*job_servers]
+
+      @uniq = opts.delete(:uniq)
+      @opts = opts
+    end
+
+    # Run a Task or Taskset
+    def run(taskset, timeout = nil, async = false)
+      timeout ||= 0
+      use_em_stop = EM.reactor_running?
+      EM.run do
+        @taskset ||= Taskset.new
+        @taskset += Taskset.create(taskset)
+
+        job_servers = @job_servers.dup
+        @reactors.each do |reactor|
+          unless reactor.connected?
+            reactor.reconnect true
+            reactor.keep_connected = async
+            reactor.callback { create_job(@taskset.shift, reactor) }
+            job_servers.delete reactor.to_s
+          else
+            create_job(@taskset.shift, reactor)
+          end
+        end
+        job_servers.each do |hostport|
+          host, port = hostport.split(":")
+          reactor = Gearman::Evented::ClientReactor.connect(host, port, @opts)
+          reactor.keep_connected = async
+          reactor.callback { create_job(@taskset.shift, reactor) }
+          @reactors << reactor
+        end
+
+        if timeout > 0
+          if use_em_stop
+            EM.add_timer(timeout) { EM.stop }
+          else
+            sleep timeout
+          end
+        elsif !async
+          Thread.new do
+            loop do
+              sleep 0.1
+              live = 0
+              @reactors.each {|reactor| live += 1 if reactor.connected? }
+              break if live == 0
+            end
+          end.join
+        end
+      end
+    end
+
+    private
+
+      def create_job(task, reactor = nil)
+        return unless task
+        reactor ||= @reactors[rand(@reactors.size)]
+        unless reactor.connected?
+          log "create_job: server #{reactor} not connected"
+          EM.next_tick { create_job(task) }
+          return
+        end
+
+        reactor.submit_job(task) {|handle| create_job(@taskset.shift) }
+      end
+
+      def log(msg)
+        Gearman::Util.log(msg)
+      end
+
+  end
+end