rcelery 1.0.0

data/lib/rcelery/version.rb

@@ -0,0 +1,6 @@
+module RCelery
+  unless const_defined?(:VERSION)
+    spec = Gem::Specification.load(File.dirname(__FILE__) + '/../../rcelery.gemspec')
+    VERSION = spec.version
+  end
+end

data/lib/rcelery/worker.rb

@@ -0,0 +1,76 @@
+require 'singleton'
+require 'forwardable'
+require 'json'
+require 'benchmark'
+
+module RCelery
+  class Worker
+    include Task::States
+
+    class << self
+      extend Forwardable
+      def_delegators :start, :stop, :join
+    end
+
+    def initialize
+      @heartbeat = 60
+      @poll_interval = 0.5
+
+      @ident = 'rcelery'
+      @version  = RCelery::VERSION
+      @system = RUBY_PLATFORM
+    end
+
+    def start(pool)
+      RCelery::Events.worker_online(@ident, @version, @system)
+      @pool = pool
+      start_heartbeat
+      subscribe
+    end
+
+    def subscribe
+      loop do
+        consume @pool.poll
+      end
+    end
+
+    def stop
+      stop_heartbeat
+      RCelery::Events.worker_offline(@ident, @version, @system)
+    end
+
+  private
+    def start_heartbeat
+      @heartbeat_timer = EM.add_periodic_timer(@heartbeat) do
+        RCelery::Events.worker_heartbeat(@ident, @version, @system)
+      end
+    end
+
+    def stop_heartbeat
+      @heartbeat_timer.cancel unless @heartbeat_timer.nil?
+    end
+
+    def consume(data)
+      message = data[:message]
+      header = data[:header]
+
+      RCelery::Events.task_started(message['id'], Process.pid)
+
+      runner = nil
+      runtime = Benchmark.realtime do
+        runner = Task.execute(message)
+      end
+
+      case runner.status
+        when SUCCESS
+          RCelery::Events.task_succeeded(message['id'], runner.result, runtime)
+        when RETRY
+          RCelery::Events.task_retried(message['id'], runner.result, runner.result.backtrace)
+        when FAILURE
+          RCelery::Events.task_failed(message['id'], runner.result, runner.result.backtrace)
+      end
+
+      header.ack
+    end
+  end
+end

data/lib/rcelery.rb

@@ -0,0 +1,86 @@
+require 'amqp'
+require 'rcelery/task'
+require 'rcelery/events'
+require 'rcelery/worker'
+require 'rcelery/configuration'
+require 'rcelery/daemon'
+require 'rcelery/task_support'
+require 'rcelery/async_result'
+require 'rcelery/eager_result'
+require 'rcelery/version'
+
+require 'rcelery/railtie' if defined?(Rails::Railtie)
+
+module RCelery
+  @running = false
+
+  def self.start(config = {})
+    config = Configuration.new(config) if config.is_a?(Hash)
+    @config = config
+
+    @application = config.application
+
+    unless eager_mode?
+      if AMQP.connection.nil? || !AMQP.connection.connected?
+        @thread = Thread.new { AMQP.start(config.to_hash) }
+      end
+
+      channel = RCelery.channel
+      @exchanges = {
+        :request => channel.direct('celery', :durable => true),
+        :result => channel.direct('celeryresults', :durable => true, :auto_delete => true),
+        :event => channel.topic('celeryev', :durable => true)
+      }
+      @queue  = channel.queue(RCelery.queue_name, :durable => true).bind(
+        exchanges[:request], :routing_key => RCelery.queue_name)
+    end
+
+    @running = true
+
+    self
+  end
+
+  def self.stop
+    AMQP.stop { EM.stop } unless eager_mode?
+    @channel = nil
+    @running = false
+    @queue = nil
+    @exchanges = nil
+    @thread.kill unless eager_mode?
+    @thread = nil
+  end
+
+  def self.channel
+    @channel ||= AMQP::Channel.new
+  end
+
+  def self.thread
+    @thread
+  end
+
+  def self.queue_name
+    "rcelery.#{@application}"
+  end
+
+  def self.running?
+    @running
+  end
+
+  def self.queue
+    @queue
+  end
+
+  def self.exchanges
+    @exchanges
+  end
+
+  def self.eager_mode?
+    @config.eager_mode if @config
+  end
+
+  def self.publish(exchange, message, options = {})
+    options[:routing_key] ||= queue_name
+    options[:content_type] = 'application/json'
+    exchanges[exchange].publish(message.to_json, options)
+  end
+end

data/rails/init.rb

@@ -0,0 +1,4 @@
+require 'rcelery/rails'
+
+RCelery::Rails.initialize
+

data/rcelery.gemspec

@@ -0,0 +1,28 @@
+Gem::Specification.new do |s|
+  s.name = 'rcelery'
+  s.summary = 'Ruby implementation of the Python Celery library.'
+  s.version = '1.0.0'
+
+  ignore = ['.gitignore']
+  s.files = `git ls-files`.split("\n") - ignore
+  s.test_files = `git ls-files -- spec/*`.split("\n")
+  s.executables << 'rceleryd'
+  s.require_paths = ['lib']
+
+  s.authors = ['John MacKenzie', 'Kris Schultz', 'Nat Williams']
+  s.homepage = 'http://github.com'
+  s.email = 'oss@leapfrogdevelopment.com'
+
+  s.add_dependency('amqp', '~> 0.7.3')
+  s.add_dependency('uuid', '~> 2.0')
+  s.add_dependency('json', '~> 1.0')
+  s.add_dependency('configtoolkit', '~> 2.3')
+  s.add_dependency('qusion', '~> 0.1.9')
+
+  s.add_development_dependency('rspec', '~> 2.6')
+  s.add_development_dependency('rake', '~> 0.9.2')
+  s.add_development_dependency('rr', '~> 1.0')
+  s.add_development_dependency('SystemTimer', '~> 1.1')
+  s.add_development_dependency('foreman', '~> 0.20')
+end
+

data/spec/bin/ci

@@ -0,0 +1,81 @@
+#! /bin/bash
+
+function install_gem_dependencies {
+  bundle install
+}
+
+function install_python_dependencies {
+  pushd spec/integration/python_components
+    virtualenv --no-site-packages .
+    bin/pip install -r requirements.txt
+  popd
+}
+
+function run_rabbit {
+  pushd spec/integration
+    bundle exec foreman start rabbit &
+  popd
+
+  export RABBITPID=$!
+  rabbitmqctl -q -n $RABBITMQ_NODENAME wait
+  rabbitmqctl -q -n $RABBITMQ_NODENAME add_vhost /integration
+  rabbitmqctl -q -n $RABBITMQ_NODENAME set_permissions -p /integration guest ".*" ".*" ".*"
+}
+
+function run_daemons {
+  pushd spec/integration
+    bundle exec foreman start -c 'rabbit=0,celeryd=1,rceleryd=1' &
+  popd
+
+  export DAEMONSPID=$!
+}
+
+function setup {
+  export RCELERY_PORT=${RCELERY_PORT:-5672}
+  export RCELERY_HOST=${RCELERY_HOST:-localhost}
+  export RCELERY_VHOST=${RCELERY_VHOST:-/integration}
+  export RABBITMQ_NODENAME=rcelery@localhost
+  export RABBITMQ_NODE_PORT=$RCELERY_PORT
+  export RABBITMQ_LOG_BASE=var/log
+  export RABBITMQ_MNESIA_BASE=var/mnesia
+
+  mkdir -p spec/integration/var/log
+
+  install_gem_dependencies
+  install_python_dependencies
+  # run_rabbit
+  run_daemons
+}
+
+function teardown {
+  # kill $RABBITPID
+  kill $DAEMONSPID
+}
+
+function break_on_fail {
+  if [ $? -ne 0 ]; then
+    teardown
+    exit 1
+  fi
+}
+
+function run_units {
+  bundle exec rake spec:unit
+  break_on_fail
+}
+
+function run_integrations {
+  export RCELERY_APPLICATION=integration
+  bundle exec rake spec:integration:ruby_worker
+  break_on_fail
+
+  export RCELERY_APPLICATION=python.integration
+  bundle exec rake spec:integration:python_worker
+  break_on_fail
+}
+
+setup
+  run_units
+  run_integrations
+teardown
+

data/spec/integration/Procfile

@@ -0,0 +1,4 @@
+rabbit: bin/rabbitmq-server
+rceleryd: bin/rceleryd --tasks tasks --vhost=$RCELERY_VHOST --host=$RCELERY_HOST --port=$RCELERY_PORT --application integration --workers 4
+celeryd: bin/celeryd -Q rcelery.python.integration
+

data/spec/integration/bin/celeryd

@@ -0,0 +1,4 @@
+#! /bin/bash
+
+cd python_components && bin/celeryd $*
+

data/spec/integration/bin/rabbitmq-server

@@ -0,0 +1,4 @@
+#! /bin/bash
+
+rabbitmq-server $*
+

data/spec/integration/bin/rceleryd

@@ -0,0 +1,4 @@
+#! /bin/bash
+
+bundle exec rceleryd $*
+

data/spec/integration/python_components/celery_client.py

@@ -0,0 +1,5 @@
+from tasks import multiply
+
+result = multiply.delay(5,4)
+print result.get(4)
+

data/spec/integration/python_components/celery_deferred_client.py

@@ -0,0 +1,10 @@
+from datetime import datetime
+
+from tasks import multiply
+
+now = datetime.now()
+run_at = now.replace(second = (now.second + 5) % 60)
+
+result = multiply.apply_async(args=[5,5], eta=run_at)
+print result.get(10)
+

data/spec/integration/python_components/celeryconfig.py

@@ -0,0 +1,26 @@
+from os import environ
+
+BROKER_HOST = environ.get('RCELERY_HOST', "localhost")
+BROKER_PORT = environ.get('RCELERY_PORT', 5672)
+BROKER_USER = environ.get('RCELERY_USERNAME', "guest")
+BROKER_PASSWORD = environ.get('RCELERY_PASSWORD', "guest")
+BROKER_VHOST = environ.get('RCELERY_VHOST', "/integration")
+
+CELERY_RESULT_BACKEND = "amqp"
+CELERY_RESULT_PERSISTENT = True
+
+CELERY_QUEUES = {
+    "rcelery.integration": {"exchange": "celery",
+                            "routing_key": "rcelery.integration"},
+    "rcelery.python.integration": {"exchange": "celery",
+                            "routing_key": "rcelery.python.integration"}
+}
+
+CELERY_DEFAULT_QUEUE = "rcelery.integration"
+
+CELERY_RESULT_SERIALIZER = "json"
+CELERY_TASK_SERIALIZER = "json"
+CELERY_AMQP_TASK_RESULT_EXPIRES = 3600
+
+CELERY_IMPORTS = ("tasks", )
+CELERY_SEND_EVENTS = True

data/spec/integration/python_components/requirements.txt

@@ -0,0 +1,9 @@
+amqplib==1.0.0
+anyjson==0.3.1
+celery==2.3.1
+importlib==1.0.2
+kombu==1.2.1
+ordereddict==1.1
+pyparsing==1.5.6
+python-dateutil==1.5
+wsgiref==0.1.2

data/spec/integration/python_components/tasks.py

@@ -0,0 +1,13 @@
+import time
+
+from celery.task import task
+
+
+@task(name='r_celery.integration.add')
+def add(a, b):
+    return a + b
+
+@task(name='r_celery.integration.multiply')
+def multiply(a, b):
+    return a * b
+

data/spec/integration/ruby_client_python_worker_spec.rb

@@ -0,0 +1,17 @@
+require 'integration/spec_helper'
+require 'system_timer'
+
+describe 'Ruby Client' do
+  include Tasks
+
+  it 'is able to talk to a python worker' do
+    result = add.delay(5,10)
+    result.wait.should == 15
+  end
+
+  it 'can send tasks scheduled in the future to python workers' do
+    result = add.apply_async(:args => [5,3], :eta => Time.now + 5)
+
+    result.wait.should == 8
+  end
+end

data/spec/integration/ruby_worker_spec.rb

@@ -0,0 +1,115 @@
+require 'integration/spec_helper'
+require 'system_timer'
+
+describe 'Ruby Worker' do
+  include Tasks
+
+  it 'is able to consume messages posted by the ruby client' do
+    result = subtract.delay(16,9)
+    result.wait.should == 7
+  end
+
+  it 'is able to consume messages posted by the python client' do
+    result = `./spec/integration/python_components/bin/python ./spec/integration/python_components/celery_client.py`.to_i
+    result.should == 20
+  end
+
+  it 'is able to consume messages with an eta from the python client' do
+    result = `./spec/integration/python_components/bin/python ./spec/integration/python_components/celery_deferred_client.py`.to_i
+    result.should == 25
+  end
+
+  it 'will defer tasks scheduled for the future' do
+    sleep_result = sleeper.apply_async(:args => 5, :eta => Time.now + 5)
+
+    SystemTimer.timeout(4) do
+      result = subtract.delay(20, 1)
+      result.wait.should == 19
+    end
+
+    sleep_result.wait.should == 'FINISH'
+  end
+
+  it 'will retry a failed task' do
+    task_id = UUID.generate
+    stub(UUID).generate { task_id }
+
+    channel = RCelery.channel
+    event_queue = channel.queue('retries', :durable => true).bind(
+      RCelery.exchanges[:event], :routing_key => 'task.retried')
+
+    retries = 0
+    event_queue.subscribe do |header, payload|
+      event = JSON.parse(payload)
+      retries += 1 if event['uuid'] == task_id
+    end
+
+    result = retrier.delay(2)
+
+    result.wait.should == 'FINISH'
+    retries.should == 2
+
+    event_queue.unsubscribe
+  end
+
+  it 'will not exceed the max retries (default 3)' do
+    task_id = UUID.generate
+    stub(UUID).generate { task_id }
+
+    channel = RCelery.channel
+    event_queue = channel.queue('failed_retries', :durable => true).bind(
+      RCelery.exchanges[:event], :routing_key => 'task.retried')
+
+    retries = 0
+    event_queue.subscribe do |header, payload|
+      event = JSON.parse(payload)
+      retries += 1 if event['uuid'] == task_id
+    end
+
+    result = retrier.delay(4)
+
+    result.wait.should =~ /MaxRetriesExceeded/
+    retries.should == 3
+
+    event_queue.unsubscribe
+  end
+
+  it 'is able to concurrently process tasks' do
+    sleep_result = sleeper.delay(5)
+    sleep(1)
+
+    SystemTimer.timeout(3) do
+      subtract_result = subtract.delay(20,1)
+      subtract_result.wait.should == 19
+    end
+
+    sleep_result.wait.should == "FINISH"
+  end
+
+  it 'is able to concurrently process many of the same task' do
+    add_result1 = add.delay(5,5)
+    add_result2 = add.delay(6,7)
+    add_result3 = add.delay(7,9)
+
+    add_result1.wait.should == 10
+    add_result2.wait.should == 13
+    add_result3.wait.should == 16
+  end
+
+  it 'is able to concurrently retry many of the same task' do
+    result1 = noop_retry.delay(1)
+    result2 = noop_retry.delay(2)
+    result3 = noop_retry.delay(3)
+    result4 = noop_retry.delay(4)
+    result5 = noop_retry.delay(5)
+    result6 = noop_retry.delay(6)
+
+    result1.wait.should == 1
+    result2.wait.should == 2
+    result3.wait.should == 3
+    result4.wait.should == 4
+    result5.wait.should == 5
+    result6.wait.should == 6
+  end
+end
+

data/spec/integration/spec_helper.rb

@@ -0,0 +1,43 @@
+require 'rubygems'
+require 'bundler'
+
+Bundler.setup
+
+require 'rspec'
+require 'rcelery'
+require 'spec/integration/tasks'
+require 'timeout'
+
+
+module Support
+  def self.config
+    {
+      :application => ENV['RCELERY_APPLICATION'] || 'integration',
+      :host =>        ENV['RCELERY_HOST'] || 'localhost',
+      :port =>        ENV['RCELERY_PORT'] || 5672,
+      :vhost =>       ENV['RCELERY_VHOST'] || '/integration',
+      :username =>    ENV['RCELERY_USERNAME'] || 'guest',
+      :password =>    ENV['RCELERY_PASSWORD'] || 'guest',
+      :worker_count => ENV['RCELERY_WORKERS'] || 2
+    }
+  end
+end
+
+RSpec.configure do |config|
+  config.mock_with :rr
+
+  config.before :all do
+    RCelery.start(Support.config)
+  end
+
+  config.after :each do
+    RCelery.queue.purge()
+  end
+
+  config.around :each do |example|
+    Timeout.timeout(15) do
+      example.run
+    end
+  end
+end
+

data/spec/integration/tasks.rb

@@ -0,0 +1,37 @@
+module Tasks
+  include RCelery::TaskSupport
+
+  task(:name => 'r_celery.integration.subtract', :ignore_result => false)
+  def subtract(a,b)
+    a - b
+  end
+
+  task(:name => 'r_celery.integration.multiply', :ignore_result => false)
+  def multiply(a,b)
+    a * b
+  end
+
+  task(:name => 'r_celery.integration.add', :ignore_result => false)
+  def add(a,b)
+    a + b
+  end
+
+  task(:name => 'r_celery.integration.sleep', :ignore_result => false)
+  def sleeper(t)
+    sleep(t.to_i)
+    'FINISH'
+  end
+
+  task(:name => 'r_celery.integration.retry', :ignore_result => false)
+  def retrier(retries = 0)
+    retrier.retry(:args => [retries-1], :eta => Time.now)  unless retries.zero?
+    'FINISH'
+  end
+
+  task(:name => 'r_celery.integration.noop_retry', :ignore_result => false)
+  def noop_retry(a)
+    noop_retry.retry(:eta => Time.now)
+  rescue RCelery::Task::MaxRetriesExceededError
+    a
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,34 @@
+require 'rubygems'
+require 'bundler'
+
+Bundler.setup
+
+require 'rspec'
+require 'rcelery'
+
+module AMQPMock
+  def stub_amqp
+    queue = stub!.bind { queue }.subject
+    stub(queue).subscribe { queue }
+    stub(queue).unsubscribe { queue }
+    channel = stub!.direct.subject
+    stub(channel).topic
+    stub(channel).queue { queue }
+
+    stub(RCelery).channel{ channel }
+
+    [channel, queue]
+  end
+end
+
+RSpec.configure do |config|
+  config.mock_with :rr
+
+  config.before :all do
+    stub(AMQP).start
+    stub(AMQP).stop
+  end
+
+  config.include AMQPMock
+end
+