queueing_rabbit 0.1.0.rc1

data/lib/queueing_rabbit.rb

@@ -0,0 +1,67 @@
+require "queueing_rabbit/version"
+require "queueing_rabbit/callbacks"
+require "queueing_rabbit/configuration"
+require "queueing_rabbit/logging"
+require "queueing_rabbit/serializer"
+require "queueing_rabbit/client/callbacks"
+require "queueing_rabbit/client/amqp"
+require "queueing_rabbit/client/bunny"
+require "queueing_rabbit/job"
+require "queueing_rabbit/worker"
+# require "queueing_rabbit/new_relic"
+
+module QueueingRabbit
+  extend self
+  extend Logging
+  extend Callbacks
+  extend Configuration
+
+  class QueueingRabbitError < Exception; end
+  class JobNotFoundError < QueueingRabbitError; end
+  class JobNotPresentError < QueueingRabbitError; end
+
+  attr_accessor :logger, :client
+
+  def connect
+    @connection ||= client.connect
+  end
+
+  def connection
+    @connection ||= connect
+  end
+
+  def drop_connection
+    @connection = nil
+  end
+
+  def enqueue(job, arguments = {})
+    info "enqueueing job #{job} with arguments: #{arguments.inspect}."
+
+    connection.open_channel(job.channel_options) do |c, _|
+      connection.define_queue(c, job.queue_name, job.queue_options)
+      connection.enqueue(c, job.queue_name, arguments)
+    end
+
+    true
+  end
+  alias_method :publish, :enqueue
+
+  def queue_size(job)
+    size = 0
+    connection.open_channel(job.channel_options) do |c, _|
+      queue = connection.define_queue(c, job.queue_name, job.queue_options)
+      size = connection.queue_size(queue)
+    end
+    size
+  end
+
+  def purge_queue(job)
+    connection.open_channel(job.channel_options) do |c, _|
+      connection.define_queue(c, job.queue_name, job.queue_options).purge
+    end
+    true
+  end
+end
+
+QueueingRabbit.client = QueueingRabbit.default_client
+QueueingRabbit.logger = Logger.new(STDOUT)

data/lib/tasks/queueing_rabbit.rake

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../../lib'
+require 'queueing_rabbit/tasks'

data/queueing_rabbit.gemspec

@@ -0,0 +1,49 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/queueing_rabbit/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Artem Chistyakov"]
+  gem.email         = ["chistyakov.artem@gmail.com"]
+  gem.summary       = %q{QueueingRabbit is an AMQP-based queueing system}
+  gem.homepage      = "https://github.com/temochka/queueing_rabbit"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "queueing_rabbit"
+  gem.require_paths = ["lib"]
+  gem.version       = QueueingRabbit::VERSION
+
+  gem.extra_rdoc_files  = [ "LICENSE", "README.md" ]
+  gem.rdoc_options      = ["--charset=UTF-8"]
+
+  gem.add_dependency "amqp",  ">= 0.9.0"
+  gem.add_dependency "bunny", ">= 0.9.0.pre7"
+  gem.add_dependency "rake",  ">= 0"
+
+  gem.description   = <<description
+    QueueingRabbit is a Ruby library providing convenient object-oriented syntax
+    for managing background jobs using AMQP. All jobs' argumets are serialized
+    to JSON and transfered to jobs using AMQP message payload. The library
+    implements amqp and bunny gems as adapters making it possible to use
+    synchronous publishing and asynchronous consuming, which might be useful for
+    Rails app running on non-EventMachine based application servers (i. e.
+    Passenger).
+
+    Any Ruby class or Module can be transformed into QueueingRabbit's background
+    job by including QueueingRabbit::Job module. It is also possible to inherit
+    your class from QueueingRabbit::AbstractJob abstract class.
+
+    The library is bundled with a Rake task which is capable of starting a
+    worker processing a specified list of jobs.
+
+    To achieve the required simplicity the gem doesn't try to support all
+    features of AMQP protocol. It uses a restricted subset instead:
+
+    * Only a single direct exchange is used
+    * Every job is consumed using a separate channel
+    * All jobs are consumed with acknowledgements
+    * ACK is only sent to the broker if a job was processed successfully
+    * Currently all messages are published with persistent option
+description
+end

data/spec/integration/asynchronous_publishing_and_consuming_spec.rb

@@ -0,0 +1,62 @@
+require 'spec_helper'
+require 'integration/jobs/print_line_job'
+
+describe "Asynchronous publishing and consuming example" do
+  include_context "Evented spec"
+  include_context "StringIO logger"
+
+  let(:job) { PrintLineJob }
+
+  before(:all) { QueueingRabbit.client = QueueingRabbit::Client::AMQP }
+  after(:all) { QueueingRabbit.client = QueueingRabbit.default_client }
+
+  context "basic consuming" do
+    let(:connection) { QueueingRabbit.connection }
+    let(:worker) { QueueingRabbit::Worker.new(job.to_s) }
+    let(:io) { StringIO.new }
+    let(:line) { "Hello, world!" }
+
+    before(:each) do
+      QueueingRabbit.drop_connection
+      PrintLineJob.io = io
+    end
+
+    it "processes enqueued jobs" do
+      em {
+        QueueingRabbit.connect
+        queue_size = nil
+
+        delayed(0.5) {
+          3.times { QueueingRabbit.enqueue(job, :line => line) }
+        }
+
+        delayed(1.0) { worker.work }
+
+        delayed(2.0) {
+          connection.open_channel do |c, _|
+            connection.define_queue(c, :print_line_job, job.queue_options).status do |s, _|
+              queue_size = s
+            end
+          end
+        }
+
+        done(2.5) { queue_size.should == 0 }
+      }
+    end
+
+    it "actually outputs the line" do
+      em {
+        QueueingRabbit.connect
+
+        delayed(0.5) { QueueingRabbit.enqueue(job, :line => line) }
+
+        delayed(1.0) { worker.work }
+
+        done(2.0) {
+          io.string.should include(line)
+        }
+      }
+    end
+  end
+end
+

data/spec/integration/jobs/print_line_job.rb

@@ -0,0 +1,17 @@
+class PrintLineJob
+  extend QueueingRabbit::Job
+
+  class << self
+    attr_writer :io
+  end
+
+  queue :print_line_job, :durable => true
+
+  def self.perform(arguments = {})
+    self.io.puts arguments[:line]
+  end
+
+  def self.io
+    @io ||= STDOUT
+  end
+end

data/spec/integration/synchronous_publishing_and_asynchronous_consuming_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+require 'integration/jobs/print_line_job'
+
+describe "Synchronous publishing and asynchronous consuming example" do
+  include_context "StringIO logger"
+  include_context "Evented spec"
+
+  let(:job) { PrintLineJob }
+  let(:line) { "Hello, world!" }
+
+  after(:all) { QueueingRabbit.client = QueueingRabbit.default_client }
+
+  context "when a message is published synchronously" do
+    before do
+      QueueingRabbit.publish(job, line: line)
+      QueueingRabbit.drop_connection
+    end
+
+    context "and being consumed asynchornously" do
+      let(:worker) { QueueingRabbit::Worker.new(job.to_s) }
+      let(:io) { StringIO.new }
+
+      before do
+        job.io = io
+      end
+
+      it "works" do
+        em {
+          worker.work
+
+          done(1.0) {
+            io.string.should include(line)
+          }
+        }
+      end
+    end
+  end
+
+end

data/spec/integration/synchronous_publishing_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+require 'integration/jobs/print_line_job'
+
+describe "Synchronous publishing example" do
+  include_context "StringIO logger"
+
+  let(:job) { PrintLineJob }
+
+  context "when publishing a message" do
+    after do
+      QueueingRabbit.purge_queue(job)
+    end
+
+    let(:publishing) {
+      -> { QueueingRabbit.publish(job, line: "Hello, World!") }
+    }
+
+    it 'affects the queue size' do
+      expect { 5.times { publishing.call } }
+        .to change{QueueingRabbit.queue_size(job)}.by(5)
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,26 @@
+$:.unshift File.expand_path('..', __FILE__)
+$:.unshift File.expand_path('../../lib', __FILE__)
+
+require 'rubygems'
+require 'bundler'
+Bundler.setup(:test)
+
+require 'rspec'
+require 'rspec/autorun'
+require 'evented-spec'
+require 'support/shared_contexts'
+require 'support/shared_examples'
+
+require 'queueing_rabbit'
+
+RSpec.configure do |config|
+  config.before(:each) {
+    QueueingRabbit.drop_connection
+  }
+
+  QueueingRabbit.configure do |qr|
+    qr.amqp_uri = "amqp://guest:guest@localhost:5672"
+    qr.amqp_exchange_name = "queueing_rabbit_test"
+    qr.amqp_exchange_options = {:durable => true}
+  end
+end

data/spec/support/shared_contexts.rb

@@ -0,0 +1,17 @@
+require 'stringio'
+
+shared_context "StringIO logger" do
+
+  before(:all) do
+    QueueingRabbit.logger = Logger.new(StringIO.new)
+  end
+
+  after(:all) do
+    QueueingRabbit.logger = Logger.new(STDOUT)
+  end
+
+end
+
+shared_context "Evented spec" do
+  include EventedSpec::SpecHelper
+end

data/spec/support/shared_examples.rb

@@ -0,0 +1,60 @@
+shared_examples :client do
+  
+  describe '#define_queue' do
+    let(:exchange) { mock }
+    let(:channel) { mock }
+    let(:queue) { mock }
+    let(:queue_name) { "test_queue_name" }
+    let(:routing_keys) { [:test_job] }
+    let(:options) { {:durable => false, :routing_keys => routing_keys} }
+
+    before do
+      client.stub(:exchange => exchange)
+      channel.should_receive(:queue).with(queue_name, options)
+                                    .and_yield(queue)
+      queue.should_receive(:bind)
+           .with(exchange, :routing_key => routing_keys.first.to_s).ordered
+      queue.should_receive(:bind)
+           .with(exchange, :routing_key => queue_name).ordered
+    end
+
+    it "defines a queue and binds it to its name and the given routing keys" do
+      client.define_queue(channel, queue_name, options)
+    end
+  end
+
+  describe '#define_exchange' do
+    let(:channel) { mock }
+    let(:options) { {:durable => true} }
+
+    before do
+      channel.should_receive(:direct)
+             .with(QueueingRabbit.amqp_exchange_name,
+                   QueueingRabbit.amqp_exchange_options.merge(options))
+    end
+
+    it 'defines a new AMQP direct exchange with given name and options' do
+      client.define_exchange(channel, options)
+    end
+  end
+
+  describe '#enqueue' do
+    let(:channel) { mock }
+    let(:exchange) { mock }
+    let(:routing_key) { :routing_key }
+    let(:payload) { {"test" => "data"} }
+
+    before do
+      client.should_receive(:exchange).with(channel).and_return(exchange)
+      exchange.should_receive(:publish).with(JSON.dump(payload),
+                                             :key => routing_key.to_s,
+                                             :persistent => true)
+    end
+
+    it "publishes a new persistent message to the used exchange with " \
+       "serialized payload and routed using given routing key" do
+      client.enqueue(channel, routing_key, payload)
+    end
+  end
+
+end

data/spec/unit/queueing_rabbit/callbacks_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+describe QueueingRabbit::Callbacks do
+  let!(:klass) { Class.new { extend QueueingRabbit::Callbacks } }
+  subject { klass }
+
+  context 'when a single callback is set for an event' do
+    let(:callback) { Proc.new {} }
+
+    before do
+      subject.setup_callback(:test, &callback)
+    end
+
+    it 'saves the callback internally' do
+      subject.instance_variable_get(:@callbacks).should include(:test)
+    end
+
+    context 'and when an event is triggered' do
+      before do
+        callback.should_receive(:call)
+      end
+
+      it 'executes the registered callback' do
+        subject.trigger_event(:test)
+      end
+    end
+  end
+
+  context 'when multiple callbacks are set for an event' do
+    let(:callback_1) { Proc.new {} }
+    let(:callback_2) { Proc.new {} }
+
+    before do
+      subject.setup_callback(:test, &callback_1)
+      subject.setup_callback(:test, &callback_2)
+    end
+
+    it 'saves the callbacks internally' do
+      subject.instance_variable_get(:@callbacks).should include(:test)
+    end
+
+    context 'and when an event is triggered' do
+      before do
+        callback_1.should_receive(:call)
+        callback_2.should_receive(:call)
+      end
+
+      it 'executes all the registered callbacks' do
+        subject.trigger_event(:test)
+      end
+    end
+  end
+end

data/spec/unit/queueing_rabbit/client/amqp_spec.rb

@@ -0,0 +1,193 @@
+require 'spec_helper'
+
+describe QueueingRabbit::Client::AMQP do
+
+  include_context "StringIO logger"
+
+  let(:connection) { mock :on_tcp_connection_loss => nil, :on_recovery => nil }
+
+  before do
+    QueueingRabbit.stub(:amqp_uri => 'amqp://localhost:5672',
+                        :amqp_exchange_name => 'queueing_rabbit_test',
+                        :amqp_exchange_options => {:durable => true})
+  end
+
+  context "class" do
+    subject { QueueingRabbit::Client::AMQP }
+
+    its(:connection_options) { should include(:timeout) }
+    its(:connection_options) { should include(:heartbeat) }
+    its(:connection_options) { should include(:on_tcp_connection_failure) }
+
+    describe '.run_event_machine' do
+      context 'when the event machine reactor is running' do
+        before do
+          EM.should_receive(:reactor_running?).and_return(true)
+        end
+
+        it 'has no effect' do
+          subject.run_event_machine
+        end
+      end
+
+      context 'when the event machine reactor is not running' do
+        before do
+          EM.should_receive(:reactor_running?).and_return(false)
+          Thread.should_receive(:new).and_yield
+          EM.should_receive(:run).and_yield
+        end
+
+        it 'runs the event machine reactor in a separate thread' do
+          subject.run_event_machine
+        end
+
+        it 'triggers :event_machine_started event' do
+          QueueingRabbit.should_receive(:trigger_event)
+                        .with(:event_machine_started)
+          subject.run_event_machine
+        end
+      end
+    end
+
+    describe ".connect" do
+      before do
+        AMQP.should_receive(:connect).with(QueueingRabbit.amqp_uri)
+                                     .and_return(connection)
+        subject.should_receive(:run_event_machine)
+      end
+
+      it "creates a class' instance" do
+        subject.connect.should be_instance_of(subject)
+      end
+    end
+
+    describe '.join_event_machine_thread' do
+      let(:thread) { mock(:join => true) }
+
+      before do
+        subject.instance_variable_set(:@event_machine_thread, thread)
+      end
+
+      it "joins the thread if exists" do
+        subject.join_event_machine_thread
+      end
+    end
+  end
+
+  context "instance" do
+    let(:client) { QueueingRabbit::Client::AMQP.connect }
+    subject { client }
+
+    before do
+      EM.stub(:reactor_running? => true)
+      AMQP.stub(:connect => connection)
+      QueueingRabbit::Client::AMQP.stub(:run_event_machine => true)
+    end
+
+    it_behaves_like :client
+
+    it { should be }
+
+    describe '#listen_queue' do
+      let(:channel) { mock }
+      let(:queue_name) { mock }
+      let(:options) { mock }
+      let(:queue) { mock }
+      let(:metadata) { stub(:ack => true) }
+      let(:data) { {:data => "data"}}
+      let(:payload) { JSON.dump(data) }
+
+      before do
+        client.should_receive(:define_queue).with(channel, queue_name, options)
+              .and_return(queue)
+        queue.should_receive(:subscribe).with(:ack => true)
+                                        .and_yield(metadata, payload)
+      end
+
+      it 'listens to the queue and passes deserialized arguments to the block' do
+        client.listen_queue(channel, queue_name, options) do |arguments|
+          arguments.should == data
+        end
+      end
+
+      context "when deserialization problems occur" do
+        let(:error) { JSON::JSONError.new }
+        before do
+          client.should_receive(:deserialize).and_raise(error)
+          client.should_receive(:error)
+          client.should_receive(:debug).with(error)
+        end
+
+        it "keeps the record of the errors" do
+          client.listen_queue(channel, queue_name, options)
+        end
+
+        it "silences JSON errors" do
+          expect { client.listen_queue(channel, queue_name, options) }
+                 .to_not raise_error(error)
+        end
+      end
+    end
+
+    describe '#process_message' do
+      let(:arguments) { mock }
+
+      it "yields given arguments to the block" do
+        client.process_message(arguments) do |a|
+          a.should == arguments
+        end
+      end
+
+      it "silences all errors risen" do
+        expect { 
+          client.process_message(arguments) { |a| raise StandardError.new }
+        }.to_not raise_error(StandardError)
+      end
+
+      context "logging" do
+        let(:error) { StandardError.new }
+        before do
+          client.should_receive(:error)
+          client.should_receive(:debug).with(error)
+        end
+
+        it "keeps the record of all errors risen" do
+          client.process_message(arguments) { |a| raise error }
+        end
+      end
+    end
+
+    describe '#disconnect' do
+      before do
+        subject.should_receive(:info)
+        connection.should_receive(:close).and_yield
+        EM.should_receive(:stop)
+      end
+
+      it 'writes the log, closes connection and stops the reactor' do
+        client.disconnect
+      end
+    end
+
+    describe "#open_channel" do
+      let(:next_channel_id) { mock }
+      let(:options) { mock }
+      let(:channel) { mock }
+      let(:open_ok) { mock }
+
+      before do
+        AMQP::Channel.should_receive(:next_channel_id)
+                     .and_return(next_channel_id)
+        AMQP::Channel.should_receive(:new)
+                     .with(connection, next_channel_id, options)
+                     .and_yield(channel, open_ok)
+        channel.should_receive(:on_error)
+      end
+
+      it 'opens a new AMQP channel with given options and installs ' \
+         'on-error callback' do
+        client.open_channel(options) {}
+      end
+    end
+  end
+end

data/spec/unit/queueing_rabbit/client/bunny_spec.rb

@@ -0,0 +1,68 @@
+require 'spec_helper'
+
+describe QueueingRabbit::Client::Bunny do
+  let(:connection) { stub(:start => true) }
+
+  before do
+    QueueingRabbit.stub(:amqp_uri => 'amqp://localhost:5672',
+                        :amqp_exchange_name => 'queueing_rabbit_test',
+                        :amqp_exchange_options => {:durable => true})
+  end
+
+  context 'class' do
+    subject { QueueingRabbit::Client::Bunny }
+
+    describe '.connect' do
+      before do
+        Bunny.should_receive(:new).with(QueueingRabbit.amqp_uri)
+                                  .and_return(connection)
+      end
+
+      it "instantiates an instance of itself" do
+        subject.connect.should be_kind_of(subject)
+      end
+    end
+  end
+
+  context 'instance' do
+    let(:client) { QueueingRabbit::Client::Bunny.connect }
+    subject { client }
+
+    before do
+      Bunny.stub(:new => connection)
+    end
+
+    it_behaves_like :client
+
+    it { should be }
+
+    describe '#open_channel' do
+      let(:options) { mock }
+      let(:channel) { mock }
+
+      before do
+        connection.should_receive(:create_channel).and_return(channel)
+      end
+
+      it 'creates a channel and yields it' do
+        client.open_channel do |c, _|
+          c.should == channel
+        end
+      end
+    end
+
+    describe '#queue_size' do
+      let(:queue) { mock }
+      let(:status) { {:message_count => 42} }
+
+      before do
+        queue.should_receive(:status).and_return(status)
+      end
+
+      it "returns a number of messages in queue" do
+        client.queue_size(queue).should be_a(Fixnum)
+      end
+    end
+
+  end
+end