rcelery 1.0.0

data/Gemfile

@@ -0,0 +1,3 @@
+source 'http://rubygems.org/'
+
+gemspec

data/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2011, Leapfrog Direct Response, LLC
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of the Leapfrog Direct Response, LLC, including
+      its subsidiaries and affiliates nor the names of its
+      contributors, may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL LEAPFROG DIRECT
+RESPONSE, LLC, INCLUDING ITS SUBSIDIARIES AND AFFILIATES, BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/Rakefile

@@ -0,0 +1,34 @@
+require 'rubygems'
+require 'bundler'
+Bundler.setup
+
+require 'rake'
+require 'rspec/core/rake_task'
+require 'rake/gempackagetask'
+
+def gemspec
+  eval(File.new('rcelery.gemspec').read)
+end
+Rake::GemPackageTask.new(gemspec).define
+
+desc 'Run ruby worker integration specs'
+RSpec::Core::RakeTask.new('spec:integration:ruby_worker') do |t|
+  t.pattern = 'spec/integration/ruby_worker_spec.rb'
+  t.rspec_opts = ["--color"]
+end
+
+desc 'Run python worker integration specs'
+RSpec::Core::RakeTask.new('spec:integration:python_worker') do |t|
+  t.pattern = 'spec/integration/ruby_client_python_worker_spec.rb'
+  t.rspec_opts = ["--color"]
+end
+
+desc 'Run unit specs'
+RSpec::Core::RakeTask.new('spec:unit') do |t|
+  t.pattern = 'spec/unit/*_spec.rb'
+  t.rspec_opts = ["--color"]
+end
+
+desc 'Run all specs'
+task :spec => ['spec:unit',]
+

data/bin/rceleryd

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'rcelery'
+
+daemon = RCelery::Daemon.new(ARGV)
+daemon.run
+

data/lib/rcelery/async_result.rb

@@ -0,0 +1,23 @@
+module RCelery
+  class AsyncResult
+    def initialize(task_id)
+      @task_id = task_id
+      @queue = Task.result_queue(task_id)
+    end
+
+    def wait
+      result_value = :no_result
+      @queue.subscribe do |payload|
+        result_value = JSON.parse(payload)['result']
+      end
+
+      while(result_value == :no_result)
+        sleep(0.05)
+      end
+
+      @queue.unsubscribe
+      result_value
+    end
+  end
+end
+

data/lib/rcelery/configuration.rb

@@ -0,0 +1,25 @@
+require 'configtoolkit'
+require 'configtoolkit/hashreader'
+require 'configtoolkit/hashwriter'
+
+module RCelery
+  class Configuration < ConfigToolkit::BaseConfig
+    add_optional_param(:host, String, 'localhost')
+    add_optional_param(:port, Integer, 5672)
+    add_optional_param(:vhost, String, '/')
+    add_optional_param(:username, String, 'guest')
+    add_optional_param(:password, String, 'guest')
+    add_optional_param(:application, String, 'application')
+    add_optional_param(:worker_count, Integer, 1)
+    add_optional_param(:eager_mode, ConfigToolkit::Boolean, false)
+
+    def initialize(options = {})
+      load(ConfigToolkit::HashReader.new(options))
+    end
+
+    def to_hash
+      dump(ConfigToolkit::HashWriter.new)
+    end
+  end
+end
+

data/lib/rcelery/daemon.rb

@@ -0,0 +1,82 @@
+require 'optparse'
+require 'rcelery/pool'
+
+module RCelery
+  class Daemon
+    def initialize(args)
+      @config = RCelery::Configuration.new
+      opts = OptionParser.new do |opt|
+        opt.on('-n', '--hostname HOSTNAME', 'Hostname of the AMQP broker') do |host|
+          @config.host = host
+        end
+
+        opt.on('-p', '--port PORT', 'Port of the AMQP broker') do |port|
+          @config.port = port
+        end
+
+        opt.on('-v', '--vhost VHOST', 'Vhost of the AMQP broker') do |vhost|
+          @config.vhost = vhost
+        end
+
+        opt.on('-u', '--username USERNAME', 'Username to use during authentication with the AMQP broker') do |username|
+          @config.username = username
+        end
+
+        opt.on('-w', '--password PASSWORD', 'Password to use during authentication with the AMQP broker') do |password|
+          @config.password = password
+        end
+
+        opt.on('-a', '--application APPLICATION', 'Name of the application') do |application|
+          @config.application = application
+        end
+
+        opt.on('-t', '--tasks lib1,lib2,...', Array, 'List of libraries to require that contain task definitions') do |requires|
+          requires.each do |lib|
+            require lib
+          end
+        end
+
+        opt.on('-r', '--rails', 'Require \'config/environment\' to provide the Rails environment') do
+          require 'config/environment'
+          require 'rcelery/rails'
+          RCelery::Rails.initialize
+          ::Rails.logger.auto_flushing = true
+        end
+
+        opt.on('-W', '--workers NUMBER', 'The number of workers to launch (default 1)') do |num|
+          @config.worker_count = num
+        end
+
+        opt.on_tail('-h', '--help', 'Show this message') do
+          puts opts
+          exit
+        end
+      end
+      opts.parse!(args)
+    end
+
+    def run
+      pool = RCelery::Pool.new(@config)
+      @config.worker_count.times do
+        Thread.new do
+          @worker = RCelery::Worker.new
+          @worker.start pool
+        end
+      end
+
+      pool.start
+      trap_signals
+      RCelery.thread.join
+    end
+
+    def trap_signals
+      block = proc do
+        @worker.stop
+        RCelery.stop
+        exit
+      end
+      Signal.trap('INT', &block)
+      Signal.trap('TERM', &block)
+    end
+  end
+end

data/lib/rcelery/eager_result.rb

@@ -0,0 +1,10 @@
+module RCelery
+  class EagerResult
+    attr_accessor :wait
+
+    def initialize(value)
+      @wait = value
+    end
+
+  end
+end

data/lib/rcelery/events.rb

@@ -0,0 +1,103 @@
+require 'socket'
+
+module RCelery
+  class Events
+    def self.hostname
+      Socket.gethostname
+    end
+
+    def self.timestamp
+      Time.now.to_f
+    end
+
+    def self.task_received(uuid, name, args, kwargs, retries = 0, eta = nil)
+      RCelery.publish(:event, {
+        :type => 'task-received',
+        :uuid => uuid,
+        :name => name,
+        :args => args,
+        :kwargs => kwargs,
+        :retries => retries,
+        :eta => eta,
+        :hostname => hostname,
+        :timestamp => timestamp
+      }, :routing_key => 'task.received')
+    end
+
+    def self.task_started(uuid, pid)
+      RCelery.publish(:event, {
+        :type => 'task-started',
+        :uuid => uuid,
+        :hostname => hostname,
+        :timestamp => timestamp,
+        :pid => pid
+      }, :routing_key => 'task.started')
+    end
+
+    def self.task_succeeded(uuid, result, runtime)
+      RCelery.publish(:event, {
+        :type => 'task-succeeded',
+        :uuid => uuid,
+        :result => result,
+        :hostname => hostname,
+        :timestamp => timestamp,
+        :runtime => runtime,
+      }, :routing_key => 'task.succeeded')
+    end
+
+    def self.task_failed(uuid, exception, traceback)
+      RCelery.publish(:event, {
+        :type => 'task-failed',
+        :uuid => uuid,
+        :exception => exception,
+        :traceback => traceback,
+        :hostname => hostname,
+        :timestamp => timestamp
+      }, :routing_key => 'task.failed')
+    end
+
+    def self.task_retried(uuid, exception, traceback)
+      RCelery.publish(:event, {
+        :type => 'task-retried',
+        :uuid => uuid,
+        :exception => exception,
+        :traceback => traceback,
+        :hostname => hostname,
+        :timestamp => timestamp
+      }, :routing_key => 'task.retried')
+    end
+
+    def self.worker_online(sw_ident, sw_ver, sw_sys)
+      RCelery.publish(:event, {
+        :type => 'worker-online',
+        :sw_ident => sw_ident,
+        :sw_ver => sw_ver,
+        :sw_sys => sw_sys,
+        :hostname => hostname,
+        :timestamp => timestamp
+      }, :routing_key => 'worker.online')
+    end
+
+    def self.worker_heartbeat(sw_ident, sw_ver, sw_sys)
+      RCelery.publish(:event, {
+        :type => 'worker-heartbeat',
+        :sw_ident => sw_ident,
+        :sw_ver => sw_ver,
+        :sw_sys => sw_sys,
+        :hostname => hostname,
+        :timestamp => timestamp
+      }, :routing_key => 'worker.heartbeat')
+    end
+
+    def self.worker_offline(sw_ident, sw_ver, sw_sys)
+      RCelery.publish(:event, {
+        :type => 'worker-offline',
+        :sw_ident => sw_ident,
+        :sw_ver => sw_ver,
+        :sw_sys => sw_sys,
+        :hostname => hostname,
+        :timestamp => timestamp
+      }, :routing_key => 'worker.offline')
+    end
+  end
+end

data/lib/rcelery/pool.rb

@@ -0,0 +1,59 @@
+require 'thread'
+require 'time'
+
+module RCelery
+  class Pool
+
+    def initialize(options={})
+      @task_queue = Queue.new
+      RCelery.start(options) unless RCelery.running?
+    end
+
+    def start
+      subscribe
+    end
+
+    def subscribe
+      # amqp-client has a nice fat TODO in the delivery handler to
+      # ack if necessary; we'll just manually do it, however, the
+      # call to subscribe still needs :ack => true so the server
+      # expects our ack
+      RCelery.queue.subscribe(:ack => true) do |header, payload|
+        begin
+          message = JSON.parse(payload)
+          RCelery::Events.task_received(message['id'], message['task'], message['args'], message['kwargs'], nil, message['eta'])
+
+          if message['eta'] && Time.parse(message['eta']) > Time.now
+            defer({:message => message, :header => header})
+          else
+            @task_queue.push({:message => message, :header => header})
+          end
+        rescue JSON::ParserError
+          # not a message we care about
+          header.ack
+        end
+      end
+    end
+
+    def defer(task)
+      time_difference = (Time.parse(task[:message]['eta']) - Time.now).to_i
+      EM.add_timer(time_difference) do
+        @task_queue.push(task)
+      end
+    end
+
+    def poll
+      @task_queue.pop
+    end
+
+    def unsubscribe
+      RCelery.queue.unsubscribe
+    end
+
+    def stop
+      unsubscribe
+      RCelery.stop
+    end
+
+  end
+end

data/lib/rcelery/rails.rb

@@ -0,0 +1,29 @@
+require 'qusion'
+
+module RCelery
+  def self.thread
+    @thread ||= Qusion.thread
+  end
+
+  module Rails
+    def self.initialize
+      config_file = File.join(::Rails.root, 'config', 'rcelery.yml')
+      raw_config = nil
+
+      if File.exists?(config_file)
+        raw_config = YAML.load_file(config_file)[::Rails.env]
+      end
+
+      unless raw_config.nil?
+        config = RCelery::Configuration.new(raw_config)
+        if config.eager_mode
+          RCelery.start(config)
+        else
+          Qusion.start(config.to_hash) do
+            RCelery.start(config)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rcelery/railtie.rb

@@ -0,0 +1,11 @@
+require 'rcelery/rails'
+
+module RCelery
+  class Railtie < ::Rails::Railtie
+    railtie_name :rcelery
+
+    initializer "rcelery.rails" do |app|
+      RCelery::Rails.initialize
+    end
+  end
+end

data/lib/rcelery/task/context.rb

@@ -0,0 +1,25 @@
+module RCelery
+  class Task
+    class Context
+      def initialize(name)
+        @key = "#{name}.request"
+      end
+
+      def update(options = {})
+        Thread.current[@key] = options
+      end
+
+      def clear
+        update
+      end
+
+      def method_missing(method, *args)
+        if args.length.zero?
+          Thread.current[@key][method]
+        else
+          super(method, *args)
+        end
+      end
+    end
+  end
+end

data/lib/rcelery/task/runner.rb

@@ -0,0 +1,73 @@
+module RCelery
+  class Task
+    module States
+      SUCCESS = 'SUCCESS'.freeze
+      RETRY   = 'RETRY'.freeze
+      FAILURE = 'FAILURE'.freeze
+    end
+
+    class Runner
+      include States
+
+      attr_reader :task, :result, :status
+
+      def initialize(message)
+        @task = Task.all_tasks[message['task']]
+        @task_id = message['id']
+        @eager = message['eager'].nil? ? false : message['eager']
+
+        @args = [message['args'], message['kwargs']].flatten.compact
+        @args.pop if @args.last.is_a?(Hash) && @args.last.empty?
+
+        @queue = Task.result_queue(@task_id) unless eager_mode?
+        @task.request.update(
+          :task_id => @task_id,
+          :retries => message['retries'] || 0,
+          :args => message['args'],
+          :kwargs => message['kwargs']
+        )
+      end
+
+      def execute
+        result = @task.method.call(*@args)
+        @status = SUCCESS
+        @result = result
+        publish_result if publish_result?
+      rescue RetryError => raised
+        @result = raised
+        @status = RETRY
+      rescue Exception => raised
+        @result = raised
+        @status = FAILURE
+        publish_result if publish_result?
+      ensure
+        @task.request.clear
+      end
+
+      private
+      def publish_result
+        traceback = []
+
+        if @status == FAILURE
+          traceback = result.backtrace
+        end
+
+        RCelery.publish(:result, {
+            :result => @result,
+            :status => @status,
+            :task_id => @task_id,
+            :traceback => traceback },
+          :routing_key => @task_id.gsub('-', ''),
+          :persistent => true)
+      end
+
+      def eager_mode?
+        @eager == true
+      end
+
+      def publish_result?
+        @task.ignore_result? == false && eager_mode? == false
+      end
+    end
+  end
+end

data/lib/rcelery/task.rb

@@ -0,0 +1,118 @@
+require 'uuid'
+require 'json'
+require 'rcelery/task/runner'
+require 'rcelery/task/context'
+
+module RCelery
+  class Task
+    class RetryError < StandardError; end
+    class MaxRetriesExceededError < StandardError; end
+
+    class << self
+      attr_accessor :result_queue_expires, :max_retries
+    end
+
+    self.max_retries = 3
+    self.result_queue_expires = 3600000
+
+    attr_reader :name, :method, :request
+
+    def self.all_tasks
+      @all_tasks ||= {}
+    end
+
+    def self.execute(message)
+      runner = Runner.new(message)
+      runner.execute
+      runner
+    end
+
+    def self.result_queue(task_id)
+      queue_name = task_id.gsub('-', '')
+      RCelery.channel.queue(
+        queue_name,
+        :durable => true,
+        :auto_delete => true,
+        :arguments => {'x-expires' => result_queue_expires}
+      ).bind(
+        RCelery.exchanges[:result],
+        :routing_key => queue_name
+      )
+    end
+
+    def initialize(options = {})
+      @name = options[:name]
+      @method = options[:method]
+      @routing_key = options[:routing_key]
+      @ignore_result = options[:ignore_result].nil? ?
+        true : options[:ignore_result]
+      @request = Context.new(@name)
+    end
+
+    def delay(*args)
+      kwargs = args.pop if args.last.is_a?(Hash)
+      apply_async(:args => args, :kwargs => kwargs)
+    end
+
+    def retry(options = {})
+      args = options[:args] || request.args
+      kwargs = options[:kwargs] || request.kwargs
+      max_retries = options[:max_retries] || self.class.max_retries
+
+      if (request.retries + 1) > max_retries
+        if options[:exc]
+          raise options[:exc]
+        else
+          raise MaxRetriesExceededError
+        end
+      end
+
+      apply_async(
+        :args => args,
+        :kwargs => kwargs,
+        :task_id => request.task_id,
+        :retries => request.retries + 1,
+        :eta => options[:eta] || default_eta
+      )
+
+      raise RetryError
+    end
+
+    def apply_async(options = {})
+      task_id = options[:task_id] || UUID.generate
+      task = {
+        :id => task_id,
+        :task => @name,
+        :args => options[:args],
+        :kwargs => options[:kwargs] || {}
+      }
+      task[:eta] = options[:eta].strftime("%Y-%m-%dT%H:%M:%S") if options[:eta]
+      task[:retries] = options[:retries] if options[:retries]
+
+      if RCelery.eager_mode?
+        task[:eager] = true
+        runner = Task.execute(JSON.parse(task.to_json))
+        return (EagerResult.new(runner.result) unless ignore_result?)
+      end
+
+      pub_opts = {
+        :persistent => true,
+        :routing_key => options[:routing_key] || @routing_key
+      }
+
+      # initialize result queue first to avoid races
+      res = ignore_result? ? nil: AsyncResult.new(task_id)
+      RCelery.publish(:request, task, pub_opts)
+      res
+    end
+
+    def ignore_result?
+      @ignore_result
+    end
+
+    private
+      def default_eta
+        Time.at(Time.now + (60 * 3))
+      end
+  end
+end

data/lib/rcelery/task_support.rb

@@ -0,0 +1,56 @@
+module RCelery
+  module TaskSupport
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    def self.task_name(mod, method)
+      parts = mod.split('::') << method.to_s
+      klass_name = parts.map do |part|
+        part.gsub(/([^\a])([A-Z])/) { "#{$1}_#{$2}" }.downcase
+      end.join('.')
+    end
+
+    module ClassMethods
+      attr_accessor :current_options
+
+      def method_added(method)
+        return if @current_options.nil?
+
+        mod = self
+        klass = Class.new { include mod }
+        bound_method = klass.instance_method(method).bind(klass.new)
+
+        task_name = @current_options[:name] ||
+          TaskSupport.task_name(mod.name, method)
+
+        task = Task.new(@current_options.merge(
+          :name => task_name,
+          :method => bound_method
+        ))
+
+        # current_options must be nil'ed before we redefine
+        # the method as doing so would trigger this method
+        # again and cause an infinite loop
+        @current_options = nil
+        mod.module_eval do
+          alias_method :"_#{method}", method
+
+          define_method(method) do |*args|
+            if args.length.zero?
+              task
+            else
+              send(:"_#{method}", *args)
+            end
+          end
+        end
+
+        RCelery::Task.all_tasks[task_name] = task
+      end
+
+      def task(options = {})
+        @current_options = options
+      end
+    end
+  end
+end