xing-gearman-ruby 1.0.0

data/examples/Net/Gearman/Worker.php

@@ -0,0 +1,455 @@
+<?php
+
+/**
+ * Interface for Danga's Gearman job scheduling system
+ *
+ * PHP version 5.1.0+
+ *
+ * LICENSE: This source file is subject to the New BSD license that is 
+ * available through the world-wide-web at the following URI:
+ * http://www.opensource.org/licenses/bsd-license.php. If you did not receive  
+ * a copy of the New BSD License and are unable to obtain it through the web, 
+ * please send a note to license@php.net so we can mail you a copy immediately.
+ *
+ * @category  Net
+ * @package   Net_Gearman
+ * @author    Joe Stump <joe@joestump.net> 
+ * @copyright 2007-2008 Digg.com, Inc.
+ * @license   http://www.opensource.org/licenses/bsd-license.php New BSD License
+ * @version   CVS: $Id$
+ * @link      http://pear.php.net/package/Net_Gearman
+ * @link      http://www.danga.com/gearman/
+ */
+
+require_once 'Net/Gearman/Connection.php';
+require_once 'Net/Gearman/Job.php';
+
+/**
+ * Gearman worker class
+ *
+ * Run an instance of a worker to listen for jobs. It then manages the running
+ * of jobs, etc.
+ *
+ * <code>
+ * <?php
+ * 
+ * $servers = array(
+ *     '127.0.0.1:7003',
+ *     '127.0.0.1:7004'
+ * );
+ * 
+ * $abilities = array('HelloWorld', 'Foo', 'Bar');
+ * 
+ * try {
+ *     $worker = new Net_Gearman_Worker($servers);
+ *     foreach ($abilities as $ability) {
+ *         $worker->addAbility('HelloWorld');
+ *     }
+ *     $worker->beginWork();
+ * } catch (Net_Gearman_Exception $e) {
+ *     echo $e->getMessage() . "\n";
+ *     exit;
+ * } 
+ * 
+ * ?>
+ * </code>
+ *
+ * @category  Net
+ * @package   Net_Gearman
+ * @author    Joe Stump <joe@joestump.net> 
+ * @copyright 2007-2008 Digg.com, Inc.
+ * @license   http://www.opensource.org/licenses/bsd-license.php New BSD License
+ * @link      http://www.danga.com/gearman/
+ * @see       Net_Gearman_Job, Net_Gearman_Connection
+ */
+class Net_Gearman_Worker
+{
+    /**
+     * Pool of connections to Gearman servers
+     *
+     * @var array $conn
+     */
+    protected $conn = array();
+
+    /**
+     * Pool of retry connections
+     *
+     * @var array $conn
+     */
+    protected $retryConn = array();
+
+    /**
+     * Pool of worker abilities
+     *
+     * @var array $conn
+     */
+    protected $abilities = array();
+
+    
+    /**
+     * Callbacks registered for this worker
+     *
+     * @var array $callback
+     * @see Net_Gearman_Worker::JOB_START
+     * @see Net_Gearman_Worker::JOB_COMPLETE
+     * @see Net_Gearman_Worker::JOB_FAIL
+     */
+    protected $callback = array(
+        self::JOB_START     => array(),
+        self::JOB_COMPLETE  => array(),
+        self::JOB_FAIL      => array()
+    );
+
+    /**
+     * Unique id for this worker
+     *
+     * @var string $id
+     */
+    protected $id = "";
+
+
+    /**
+     * Callback types
+     *
+     * @const integer JOB_START    Ran when a job is started
+     * @const integer JOB_COMPLETE Ran when a job is finished
+     * @const integer JOB_FAIL     Ran when a job fails
+     */
+    const JOB_START    = 1;
+    const JOB_COMPLETE = 2;
+    const JOB_FAIL     = 3;
+
+    /**
+     * Constructor
+     *
+     * @param array $servers List of servers to connect to
+     * @param string $id     Optional unique id for this worker
+     * 
+     * @return void
+     * @throws Net_Gearman_Exception
+     * @see Net_Gearman_Connection
+     */
+    public function __construct($servers, $id = "")
+    {
+        if (!is_array($servers) && strlen($servers)) {
+            $servers = array($servers);
+        } elseif (is_array($servers) && !count($servers)) {
+            throw new Net_Gearman_Exception('Invalid servers specified');
+        }
+
+        if(empty($id)){
+            $id = "pid_".getmypid()."_".uniqid();
+        }
+
+        $this->id = $id;
+
+        foreach ($servers as $s) {
+            try {
+                $conn = Net_Gearman_Connection::connect($s);   
+
+                Net_Gearman_Connection::send($conn, "set_client_id", array("client_id" => $this->id));
+
+                $this->conn[$s] = $conn;             
+
+            } catch (Net_Gearman_Exception $e) {
+
+                $this->retryConn[$s] = time();
+            }
+        }
+
+        if (empty($this->conn)) {
+            throw new Net_Gearman_Exception(
+                "Couldn't connect to any available servers"
+            );
+        }
+    }
+
+    /**
+     * Announce an ability to the job server
+     *
+     * @param string  $ability Name of functcion/ability
+     * @param integer $timeout How long to give this job
+     *
+     * @return void
+     * @see Net_Gearman_Connection::send()
+     */
+    public function addAbility($ability, $timeout = null)
+    {
+        $call   = 'can_do';
+        $params = array('func' => $ability);
+        if (is_int($timeout) && $timeout > 0) {
+            $params['timeout'] = $timeout;
+            $call              = 'can_do_timeout';
+        }
+        
+        $this->abilities[$ability] = $timeout;
+        
+        foreach ($this->conn as $conn) {
+            Net_Gearman_Connection::send($conn, $call, $params);
+        }
+    }
+
+    /**
+     * Begin working
+     *
+     * This starts the worker on its journey of actually working. The first
+     * argument is a PHP callback to a function that can be used to monitor
+     * the worker. If no callback is provided then the worker works until it
+     * is killed. The monitor is passed two arguments; whether or not the 
+     * worker is idle and when the last job was ran.
+     *
+     * @param callback $monitor Function to monitor work
+     * 
+     * @return void
+     * @see Net_Gearman_Connection::send(), Net_Gearman_Connection::connect()
+     * @see Net_Gearman_Worker::doWork(), Net_Gearman_Worker::addAbility()
+     */
+    public function beginWork($monitor = null)
+    {
+        if (!is_callable($monitor)) {
+            $monitor = array($this, 'stopWork');
+        }
+
+        $write     = null;
+        $except    = null;
+        $working   = true;
+        $lastJob   = time();
+        $retryTime = 5;
+
+        while ($working) {
+            $sleep = true;
+            $currentTime = time();
+            
+            foreach ($this->conn as $server => $socket) {
+                try {
+                    $worked = $this->doWork($socket);
+                } catch (Net_Gearman_Exception $e) {
+                    unset($this->conn[$server]);
+                    $this->retryConn[$server] = $currentTime;
+                }
+                if ($worked) {
+                    $lastJob = time();
+                    $sleep   = false;
+                }
+            }
+
+            $idle = false;
+            if ($sleep && count($this->conn)) {
+                foreach ($this->conn as $socket) {
+                    Net_Gearman_Connection::send($socket, 'pre_sleep');
+                }
+
+                $read = $this->conn;
+                socket_select($read, $write, $except, 60);
+                $idle = (count($read) == 0);
+            }
+
+            $retryChange = false;
+            foreach ($this->retryConn as $s => $lastTry) {
+                if (($lastTry + $retryTime) < $currentTime) {
+                    try {
+                        $conn = Net_Gearman_Connection::connect($s);
+                        $this->conn[$s]         = $conn;
+                        $retryChange            = true;
+                        unset($this->retryConn[$s]);
+                        Net_Gearman_Connection::send($conn, "set_client_id", array("client_id" => $this->id));
+                    } catch (Net_Gearman_Exception $e) {
+                        $this->retryConn[$s] = $currentTime;
+                    }
+                }
+            }
+            
+            if (count($this->conn) == 0) {
+                // sleep to avoid wasted cpu cycles if no connections to block on using socket_select
+                sleep(1);
+            }
+            
+            if ($retryChange === true) {
+                // broadcast all abilities to all servers
+                foreach ($this->abilities as $ability => $timeout) {
+                    $this->addAbility($ability, $timeout);
+                }
+            }
+
+            if (call_user_func($monitor, $idle, $lastJob) == true) {
+                $working = false;
+            }
+        }
+    }
+
+    /**
+     * Listen on the socket for work
+     *
+     * Sends the 'grab_job' command and then listens for either the 'noop' or
+     * the 'no_job' command to come back. If the 'job_assign' comes down the
+     * pipe then we run that job. 
+     *
+     * @param resource $socket The socket to work on 
+     * 
+     * @return boolean Returns true if work was done, false if not
+     * @throws Net_Gearman_Exception
+     * @see Net_Gearman_Connection::send()
+     */
+    protected function doWork($socket)
+    {
+        Net_Gearman_Connection::send($socket, 'grab_job');
+
+        $resp = array('function' => 'noop');
+        while (count($resp) && $resp['function'] == 'noop') {
+            $resp = Net_Gearman_Connection::blockingRead($socket);
+        } 
+
+        if (in_array($resp['function'], array('noop', 'no_job'))) {
+            return false;
+        }
+
+        if ($resp['function'] != 'job_assign') {
+            throw new Net_Gearman_Exception('Holy Cow! What are you doing?!');
+        }
+
+        $name   = $resp['data']['func'];
+        $handle = $resp['data']['handle'];
+        $arg    = array();
+
+        if (isset($resp['data']['arg']) && 
+            Net_Gearman_Connection::stringLength($resp['data']['arg'])) {
+            $arg = json_decode($resp['data']['arg'], true);
+            if($arg === null){
+                $arg = $resp['data']['arg'];
+            }
+        }
+
+        $job = Net_Gearman_Job::factory($name, $socket, $handle);
+        try {
+            $this->start($handle, $name, $arg);
+            $res = $job->run($arg); 
+            if (!is_array($res)) {
+                $res = array('result' => $res);
+            }
+
+            $job->complete($res);
+            $this->complete($handle, $name, $res);
+        } catch (Net_Gearman_Job_Exception $e) {
+            $job->fail(); 
+            $this->fail($handle, $name, $e); 
+        }
+
+        // Force the job's destructor to run
+        $job = null;
+
+        return true;
+    }
+
+    /**
+     * Attach a callback
+     *
+     * @param callback $callback A valid PHP callback
+     * @param integer  $type     Type of callback
+     * 
+     * @return void
+     * @throws Net_Gearman_Exception
+     */
+    public function attachCallback($callback, $type = self::JOB_COMPLETE)
+    {
+        if (!is_callable($callback)) {
+            throw new Net_Gearman_Exception('Invalid callback specified');
+        }
+
+        $this->callback[$type][] = $callback;
+    }
+
+    /**
+     * Run the job start callbacks
+     *
+     * @param string $handle The job's Gearman handle
+     * @param string $job    The name of the job
+     * @param mixed  $args   The job's argument list
+     *
+     * @return void
+     */
+    protected function start($handle, $job, $args)
+    {
+        if (!count($this->callback[self::JOB_START])) {
+            return; // No callbacks to run
+        }
+
+        foreach ($this->callback[self::JOB_START] as $callback) {
+            call_user_func($callback, $handle, $job, $args);
+        }
+    }
+
+    /**
+     * Run the complete callbacks
+     *
+     * @param string $handle The job's Gearman handle
+     * @param string $job    The name of the job
+     * @param array  $result The job's returned result
+     * 
+     * @return void
+     */
+    protected function complete($handle, $job, array $result)
+    {
+        if (!count($this->callback[self::JOB_COMPLETE])) {
+            return; // No callbacks to run
+        }
+
+        foreach ($this->callback[self::JOB_COMPLETE] as $callback) {
+            call_user_func($callback, $handle, $job, $result);
+        }
+    }
+
+    /**
+     * Run the fail callbacks
+     *
+     * @param string $handle The job's Gearman handle
+     * @param string $job    The name of the job
+     * @param object $error  The exception thrown
+     * 
+     * @return void
+     */
+    protected function fail($handle, $job, PEAR_Exception $error)
+    {
+        if (!count($this->callback[self::JOB_FAIL])) {
+            return; // No callbacks to run
+        }
+
+        foreach ($this->callback[self::JOB_FAIL] as $callback) {
+            call_user_func($callback, $handle, $job, $error);
+        }
+    }
+
+    /**
+     * Stop working
+     *
+     * @return void
+     */
+    public function endWork()
+    {
+        foreach ($this->conn as $conn) {
+            Net_Gearman_Connection::close($conn);
+        }
+    }
+
+    /**
+     * Destructor
+     *
+     * @return void
+     * @see Net_Gearman_Worker::stop()
+     */
+    public function __destruct()
+    {
+        $this->endWork();
+    }
+
+    /**
+     * Should we stop work?
+     *
+     * @return boolean
+     */
+    public function stopWork()
+    {
+        return false;
+    }
+}
+
+?>

data/examples/client.php

@@ -0,0 +1,23 @@
+<?php
+
+require_once 'Net/Gearman/Client.php';
+
+$set = new Net_Gearman_Set();
+
+function result($func, $handle, $result) {
+    var_dump($func);
+    var_dump($handle);
+    var_dump($result);
+}
+
+$task = new Net_Gearman_Task('Sleep', array(
+  'seconds' => 20
+));
+
+$task->attachCallback('result');
+$set->addTask($task);
+
+$client = new Net_Gearman_Client(array('localhost:4730', 'localhost:4731'));
+$client->runSet($set);
+
+?>

data/examples/client.rb

@@ -0,0 +1,15 @@
+require 'rubygems'
+#require 'gearman'
+require '../lib/gearman'
+Gearman::Util.debug = true
+
+servers = ['localhost:4730', 'localhost:4731']
+  
+client = Gearman::Client.new(servers)
+taskset = Gearman::TaskSet.new(client)
+
+task = Gearman::Task.new('sleep', 20)
+task.on_complete {|d| puts d }
+
+taskset.add_task(task)
+taskset.wait(100)

data/examples/scale_image.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/ruby
+
+$: << '../lib'
+require 'gearman'
+require 'optparse'
+
+Gearman::Util.debug = true
+servers = 'localhost:7003'
+format = 'PNG'
+width, height = 100, 100
+
+opts = OptionParser.new
+opts.banner = "Usage: #{$0} [options] <input> <output>"
+opts.on('-f FORMAT', '--format', 'Scaled image format') { format }
+opts.on('-h HEIGHT', '--height', 'Scaled image height') { height }
+opts.on('-s SERVERS', '--servers',
+  'Servers, comma-separated host:port') { servers }
+opts.on('-w WIDTH', '--width', 'Scaled image width') { width }
+opts.parse!
+
+if ARGV.size != 2
+  $stderr.puts opts.banner
+  exit 1
+end
+
+client = Gearman::Client.new(servers.split(','), 'example')
+taskset = Gearman::TaskSet.new(client)
+arg = [width, height, format, File.read(ARGV[0])].join("\0")
+task = Gearman::Task.new('scale_image', arg)
+task.on_complete {|d| File.new(ARGV[1],'w').write(d) }
+taskset.add_task(task)
+taskset.wait(10)

data/examples/scale_image_worker.rb

@@ -0,0 +1,34 @@
+#!/usr/bin/ruby
+
+$: << '../lib'
+require 'gearman'
+require 'optparse'
+require 'RMagick'
+
+Gearman::Util.debug = true
+servers = 'localhost:7003'
+
+opts = OptionParser.new
+opts.banner = "Usage: #{$0} [options]"
+opts.on('-s SERVERS', '--servers',
+  'Job servers, comma-separated host:port') { servers }
+opts.parse!
+
+worker = Gearman::Worker.new(servers.split(','), 'example')
+
+worker.add_ability('scale_image') do |data,job|
+  width, height, format, data = data.split("\0", 4)
+  width = width.to_f
+  height = height.to_f
+  image = Magick::Image.from_blob(data)[0]
+  orig_ratio = image.columns.to_f / image.rows
+  new_ratio = width / height
+  w = new_ratio < orig_ratio ? width : orig_ratio / new_ratio * width
+  h = new_ratio > orig_ratio ? height : new_ratio / orig_ratio * height
+  puts "Got #{image.inspect}; resizing to #{w}x#{h} #{format}"
+  image.resize!(w, h)
+  image.format = format
+  image.to_blob
+end
+
+loop { worker.work }

data/examples/server.rb

@@ -0,0 +1,13 @@
+require 'rubygems'
+require 'gearman'
+require 'gearman/server'
+require 'pp'
+
+Gearman::Util.debug = true
+w = Gearman::Server.new('localhost:4730')
+
+loop { 
+  pp "Status: ", w.status 
+  pp "Workers: ", w.workers
+  sleep 5
+}

data/examples/worker.php

@@ -0,0 +1,15 @@
+<?php
+error_reporting(E_ALL & ~E_NOTICE);
+
+require_once 'Net/Gearman/Worker.php';
+
+try {
+    $worker = new Net_Gearman_Worker(array('localhost:4730'));
+    $worker->addAbility('Sleep');
+    $worker->beginWork();
+} catch (Net_Gearman_Exception $e) {
+    echo $e->getMessage() . "\n";
+    exit;
+}
+
+?>

data/examples/worker.rb

@@ -0,0 +1,23 @@
+require 'rubygems'
+#require 'gearman'
+require '../lib/gearman'
+
+Gearman::Util.debug = true
+
+servers = ['localhost:4730', 'localhost:4731']
+w = Gearman::Worker.new(servers)
+
+# Add a handler for a "sleep" function that takes a single argument, the
+# number of seconds to sleep before reporting success.
+w.add_ability('sleep') do |data,job|
+ seconds = data
+ (1..seconds.to_i).each do |i|
+   sleep 1
+   print i
+   # Report our progress to the job server every second.
+   job.report_status(i, seconds)
+ end
+ # Report success.
+ true
+end
+loop { w.work }

data/gearman-ruby.gemspec

@@ -0,0 +1,81 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{gearman-ruby}
+  s.version = "1.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Daniel Erat", "Ladislav Martincik"]
+  s.date = %q{2009-06-25}
+  s.description = %q{Library for the Gearman distributed job system}
+  s.email = %q{ladislav.martincik@xing.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README"
+  ]
+  s.files = [
+    ".gitignore",
+     "LICENSE",
+     "README",
+     "Rakefile",
+     "VERSION.yml",
+     "examples/Net/Gearman/Client.php",
+     "examples/Net/Gearman/Connection.php",
+     "examples/Net/Gearman/Exception.php",
+     "examples/Net/Gearman/Job.php",
+     "examples/Net/Gearman/Job/Common.php",
+     "examples/Net/Gearman/Job/Exception.php",
+     "examples/Net/Gearman/Job/Sleep.php",
+     "examples/Net/Gearman/Manager.php",
+     "examples/Net/Gearman/Set.php",
+     "examples/Net/Gearman/Task.php",
+     "examples/Net/Gearman/Worker.php",
+     "examples/client.php",
+     "examples/client.rb",
+     "examples/scale_image.rb",
+     "examples/scale_image_worker.rb",
+     "examples/server.rb",
+     "examples/worker.php",
+     "examples/worker.rb",
+     "gearman-ruby.gemspec",
+     "lib/gearman.rb",
+     "lib/gearman/client.rb",
+     "lib/gearman/server.rb",
+     "lib/gearman/task.rb",
+     "lib/gearman/taskset.rb",
+     "lib/gearman/testlib.rb",
+     "lib/gearman/util.rb",
+     "lib/gearman/worker.rb",
+     "test/client_test.rb",
+     "test/mock_client_test.rb",
+     "test/mock_worker_test.rb",
+     "test/worker_test.rb"
+  ]
+  s.has_rdoc = true
+  s.homepage = %q{http://github.com/xing/gearman-ruby}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.1}
+  s.summary = %q{Library for the Gearman distributed job system}
+  s.test_files = [
+    "test/client_test.rb",
+     "test/mock_client_test.rb",
+     "test/mock_worker_test.rb",
+     "test/worker_test.rb",
+     "examples/client.rb",
+     "examples/scale_image.rb",
+     "examples/scale_image_worker.rb",
+     "examples/server.rb",
+     "examples/worker.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 2
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end