queueing_rabbit 0.1.0.rc1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rvmrc

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
+# Only full ruby name is supported here, for short names use:
+#     echo "rvm use 1.9.3" > .rvmrc
+environment_id="ruby-1.9.3-p392@queueing_rabbit"
+
+# Uncomment the following lines if you want to verify rvm version per project
+# rvmrc_rvm_version="1.18.18 (latest)" # 1.10.1 seams as a safe start
+# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
+#   echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
+#   return 1
+# }
+
+# First we attempt to load the desired environment directly from the environment
+# file. This is very fast and efficient compared to running through the entire
+# CLI and selector. If you want feedback on which environment was used then
+# insert the word 'use' after --create as this triggers verbose mode.
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
+  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
+then
+  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+  [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] &&
+    \. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true
+else
+  # If the environment file has not yet been created, use the RVM CLI to select.
+  rvm --create  "$environment_id" || {
+    echo "Failed to create RVM environment '${environment_id}'."
+    return 1
+  }
+fi
+
+# If you use bundler, this might be useful to you:
+# if [[ -s Gemfile ]] && {
+#   ! builtin command -v bundle >/dev/null ||
+#   builtin command -v bundle | GREP_OPTIONS= \grep $rvm_path/bin/bundle >/dev/null
+# }
+# then
+#   printf "%b" "The rubygem 'bundler' is not installed. Installing it now.\n"
+#   gem install bundler
+# fi
+# if [[ -s Gemfile ]] && builtin command -v bundle >/dev/null
+# then
+#   bundle install | GREP_OPTIONS= \grep -vE '^Using|Your bundle is complete'
+# fi

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in queueing_rabbit.gemspec
+gemspec
+
+group :test do
+  gem 'rspec', '~> 2.13.0'
+  gem 'evented-spec', '~> 0.9.0'
+end

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Artem Chistyakov
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,38 @@
+# QueueingRabbit
+
+QueueingRabbit is a Ruby library that provides a convenient object-oriented
+syntax for managing background jobs with AMQP. All jobs' argumets are
+serialized to JSON and transfered as AMQP message payloads. The library
+implements amqp and bunny gems as adapters, making it possible to use
+synchronous publishing and asynchronous consuming, which might be useful for
+Rails app running on non-EventMachine based application servers (i. e.
+Passenger).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'queueing_rabbit'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install queueing_rabbit
+
+## Usage
+
+QueueingRabbit is currently in RC1 and is not recommended for production use.
+
+The docs are coming soon, currently you can check out the examples in
+`spec/integration` dir.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,5 @@
+#!/usr/bin/env rake
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new

data/lib/queueing_rabbit/callbacks.rb

@@ -0,0 +1,31 @@
+module QueueingRabbit
+
+  module Callbacks
+
+    def before_consuming(&block)
+      setup_callback(:consuming_started, &block)
+    end
+
+    def after_consuming(&block)
+      setup_callback(:consuming_done, &block)
+    end
+
+    def on_event_machine_start(&block)
+      setup_callback(:event_machine_started, &block)
+    end
+
+    def setup_callback(event, &block)
+      @callbacks ||= {}
+      @callbacks[event] ||= []
+      @callbacks[event] << block
+    end
+
+    def trigger_event(event)
+      if @callbacks && @callbacks[event]
+        @callbacks[event].each { |c| c.call }
+      end
+    end
+
+  end
+
+end

data/lib/queueing_rabbit/client/amqp.rb

@@ -0,0 +1,148 @@
+require 'amqp'
+
+module QueueingRabbit
+
+  module Client
+
+    class AMQP
+
+      include QueueingRabbit::Serializer
+      include QueueingRabbit::Logging
+      extend  QueueingRabbit::Logging
+      extend  QueueingRabbit::Client::Callbacks
+
+      attr_reader :connection, :exchange_name, :exchange_options
+
+      define_callback :on_tcp_failure do |_|
+        fatal "unable to establish TCP connection to broker"
+        EM.stop
+      end
+
+      define_callback :on_tcp_loss do |c, _|
+        info "re-establishing TCP connection to broker"
+        c.reconnect(false, 1)
+      end
+
+      define_callback :on_tcp_recovery do
+        info "TCP connection to broker is back and running"
+      end
+
+      define_callback :on_channel_error do |ch, channel_close|
+        EM.stop
+        fatal "channel error occured: #{channel_close.reply_text}"
+      end
+
+      def self.connection_options
+        {:timeout => QueueingRabbit.tcp_timeout,
+         :heartbeat => QueueingRabbit.heartbeat,
+         :on_tcp_connection_failure => self.callback(:on_tcp_failure)}
+      end
+
+      def self.connect
+        self.run_event_machine
+
+        self.new(::AMQP.connect(QueueingRabbit.amqp_uri),
+                 QueueingRabbit.amqp_exchange_name,
+                 QueueingRabbit.amqp_exchange_options)
+      end
+
+      def self.run_event_machine
+        return if EM.reactor_running?
+
+        @event_machine_thread = Thread.new do
+          EM.run do
+            QueueingRabbit.trigger_event(:event_machine_started)
+          end
+        end
+      end
+
+      def self.join_event_machine_thread
+        @event_machine_thread.join if @event_machine_thread
+      end
+
+      def disconnect
+        info "closing AMQP broker connection..."
+
+        connection.close do
+          yield if block_given?
+
+          EM.stop if EM.reactor_running?
+        end
+      end
+
+      def define_queue(channel, queue_name, options={})
+        routing_keys = [*options.delete(:routing_keys)] + [queue_name]
+
+        channel.queue(queue_name.to_s, options) do |queue|
+          routing_keys.each do |key|
+            queue.bind(exchange(channel), :routing_key => key.to_s)
+          end
+        end
+      end
+
+      def listen_queue(channel, queue_name, options={}, &block)
+        define_queue(channel, queue_name, options)
+            .subscribe(:ack => true) do |metadata, payload|
+          begin
+            process_message(deserialize(payload), &block)
+            metadata.ack
+          rescue JSON::JSONError => e
+            error "JSON parser error occured: #{e.message}"
+            debug e
+          end
+        end
+      end
+
+      def process_message(arguments)
+        begin
+          yield arguments
+        rescue => e
+          error "unexpected error #{e.class} occured: #{e.message}"
+          debug e
+        end
+      end
+
+      def open_channel(options={})
+        ::AMQP::Channel.new(connection,
+                          ::AMQP::Channel.next_channel_id,
+                          options) do |c, open_ok|
+          c.on_error(&self.class.callback(:on_channel_error))
+          yield c, open_ok
+        end
+      end
+
+      def define_exchange(channel, options={})
+        channel.direct(exchange_name, exchange_options.merge(options))
+      end
+      alias_method :exchange, :define_exchange
+
+      def enqueue(channel, routing_key, payload)
+        exchange(channel).publish(serialize(payload), :key => routing_key.to_s,
+                                                      :persistent => true)
+      end
+      alias_method :publish, :enqueue
+
+      def queue_size(queue)
+        raise NotImplementedError
+      end
+
+    private
+
+      def setup_callbacks
+        connection.on_tcp_connection_loss(&self.class.callback(:on_tcp_loss))
+        connection.on_recovery(&self.class.callback(:on_tcp_recovery))
+      end
+
+      def initialize(connection, exchange_name, exchange_options = {})
+        @connection = connection
+        @exchange_name = exchange_name
+        @exchange_options = exchange_options
+
+        setup_callbacks
+      end
+
+    end
+
+  end
+
+end

data/lib/queueing_rabbit/client/bunny.rb

@@ -0,0 +1,62 @@
+require 'bunny'
+
+module QueueingRabbit
+
+  module Client
+
+    class Bunny
+
+      include QueueingRabbit::Serializer
+
+      attr_reader :connection, :exchange_name, :exchange_options
+
+      def self.connect
+        self.new(::Bunny.new(QueueingRabbit.amqp_uri),
+                 QueueingRabbit.amqp_exchange_name,
+                 QueueingRabbit.amqp_exchange_options)
+      end
+
+      def open_channel(options = {})
+        ch = connection.create_channel
+        yield ch, nil
+        # ch.close
+      end
+
+      def define_queue(channel, name, options = {})
+        routing_keys = [*options.delete(:routing_keys)] + [name]
+
+        channel.queue(name.to_s, options) do |q|
+          routing_keys.each { |key| q.bind(exchange, :routing_key => key.to_s) }
+        end
+      end
+
+      def enqueue(channel, routing_key, payload)
+        exchange(channel).publish(serialize(payload), :key => routing_key.to_s,
+                                                      :persistent => true)
+      end
+      alias_method :publish, :enqueue
+
+      def define_exchange(channel, options={})
+        channel.direct(exchange_name, exchange_options.merge(options))
+      end
+      alias_method :exchange, :define_exchange
+
+      def queue_size(queue)
+        queue.status[:message_count]
+      end
+
+    private
+
+      def initialize(connection, exchange_name, exchange_options)
+        @connection = connection
+        @exchange_name = exchange_name
+        @exchange_options = exchange_options
+
+        @connection.start
+      end
+
+    end
+
+  end
+
+end

data/lib/queueing_rabbit/client/callbacks.rb

@@ -0,0 +1,14 @@
+module QueueingRabbit
+  module Client
+    module Callbacks
+      def define_callback(name, &block)
+        @callbacks ||= {}
+        @callbacks[name] = block
+      end
+
+      def callback(name)
+        @callbacks[name] if @callbacks
+      end
+    end
+  end
+end

data/lib/queueing_rabbit/configuration.rb

@@ -0,0 +1,24 @@
+module QueueingRabbit
+
+  module Configuration
+    attr_accessor :amqp_uri, :amqp_exchange_name, :amqp_exchange_options
+    attr_writer :tcp_timeout, :heartbeat
+
+    def configure
+      yield self
+    end
+
+    def tcp_timeout
+      @tcp_timeout ||= 1
+    end
+
+    def heartbeat
+      @heartbeat ||= 10
+    end
+
+    def default_client
+      QueueingRabbit::Client::Bunny
+    end
+  end
+
+end

data/lib/queueing_rabbit/job.rb

@@ -0,0 +1,32 @@
+module QueueingRabbit
+  module Job
+    def queue_name
+      @queue_name ||= self.name.split('::')[-1]
+    end
+
+    def queue_options
+      @queue_options ||= {}
+    end
+
+    def queue(name, options = {})
+      @queue_name = name
+      @queue_options = options
+    end
+
+    def queue_size
+      QueueingRabbit.queue_size(self)
+    end
+
+    def channel_options
+      @channel_options ||= {}
+    end
+
+    def channel(options={})
+      @channel_options = options
+    end
+  end
+
+  class AbstractJob
+    extend Job
+  end
+end

data/lib/queueing_rabbit/logging.rb

@@ -0,0 +1,17 @@
+require "logger"
+
+module QueueingRabbit
+
+  module Logging
+
+    # Logging levels are defined at:
+    # http://www.ruby-doc.org/stdlib-1.9.3/libdoc/logger/rdoc/Logger.html
+    %w[fatal error warn info debug].each do |level|
+      define_method(level) do |message|
+        QueueingRabbit.logger.__send__(level, message) if QueueingRabbit.logger
+      end
+    end
+
+  end
+
+end

data/lib/queueing_rabbit/serializer.rb

@@ -0,0 +1,19 @@
+require 'json'
+
+module QueueingRabbit
+  module Serializer
+    def serialize(args)
+      JSON.dump(args)
+    end
+
+    def deserialize(msg)
+      symbolize_keys(JSON.parse(msg))
+    end
+
+  private
+
+    def symbolize_keys(hash)
+      hash.inject({}) { |memo, (k,v)| memo[k.to_sym] = v; memo }
+    end
+  end
+end

data/lib/queueing_rabbit/tasks.rb

@@ -0,0 +1,37 @@
+# require 'queueing_rabbit/tasks'
+# will give you the queueing_rabbit tasks
+
+namespace :queueing_rabbit do
+  task :setup
+
+  desc "Start a queueing rabbit worker"
+  task :work => :setup do
+    require 'queueing_rabbit'
+
+    if ENV['PIDFILE'] && File.exists?(ENV['PIDFILE'])
+      abort "PID file already exists. Is the worker running?"
+    end
+
+    jobs = (ENV['JOBS'] || ENV['JOB']).to_s.split(',')
+
+    begin
+      worker = QueueingRabbit::Worker.new(*jobs)
+    rescue QueueingRabbit::NoJobError
+      abort "set JOB env var, e.g. $ JOB=ExportDataJob,CompressFileJob " \
+            "rake queueing_rabbit:work"
+    end
+
+    if ENV['BACKGROUND']
+      unless Process.respond_to?('daemon')
+        abort "env var BACKGROUND is set, which requires ruby >= 1.9"
+      end
+      Process.daemon(true)
+    end
+
+    worker.use_pidfile(ENV['PIDFILE']) if ENV['PIDFILE']
+
+    worker.info "starting a new queueing_rabbit worker #{worker}"
+
+    worker.work!
+  end
+end

data/lib/queueing_rabbit/version.rb

@@ -0,0 +1,3 @@
+module QueueingRabbit
+  VERSION = "0.1.0.rc1"
+end

data/lib/queueing_rabbit/worker.rb

@@ -0,0 +1,96 @@
+module QueueingRabbit
+  class Worker
+    include QueueingRabbit::Logging
+
+    attr_accessor :jobs
+
+    def initialize(*jobs)
+      self.jobs = jobs.map { |job| job.to_s.strip }
+
+      sync_stdio
+      validate_jobs
+      constantize_jobs
+      use_asynchronous_client
+    end
+
+    def work
+      conn = QueueingRabbit.connection
+      trap_signals(conn)
+
+      jobs.each { |job| run_job(conn, job) }
+
+      QueueingRabbit.trigger_event(:consuming_started)
+    end
+
+    def work!
+      work
+      QueueingRabbit::Client::AMQP.join_event_machine_thread
+    end
+
+    def use_pidfile(filename)
+      File.open(@pidfile = filename, 'w') { |f| f << pid }
+    end
+
+    def remove_pidfile
+      File.delete(@pidfile) if @pidfile && File.exists?(@pidfile)
+    end
+
+    def pid
+      Process.pid
+    end
+
+    def to_s
+      "PID=#{pid}, JOBS=#{jobs.join(',')}"
+    end
+
+  private
+
+    def use_asynchronous_client
+      QueueingRabbit.client = QueueingRabbit::Client::AMQP
+    end
+
+    def validate_jobs
+      if jobs.nil? || jobs.empty?
+        fatal "no jobs specified to work on."
+        raise JobNotPresentError.new("No jobs specified to work on.")
+      end
+    end
+
+    def constantize_jobs
+      self.jobs = jobs.map do |job|
+        begin
+          Kernel.const_get(job)
+        rescue NameError
+          fatal "job #{job} doesn't exist."
+          raise JobNotFoundError.new("Job #{job} doesn't exist.")
+        end
+      end
+    end
+
+    def run_job(conn, job)
+      conn.open_channel(job.channel_options) do |channel, _|
+        conn.listen_queue(channel, job.queue_name, job.queue_options) do |args|
+          info "performing job #{job} with arguments #{args.inspect}"
+          job.perform(args)
+        end
+      end
+    end
+
+    def sync_stdio
+      $stdout.sync = true
+      $stderr.sync = true
+    end
+
+    def trap_signals(connection)
+      handler = Proc.new do
+        connection.disconnect {
+          QueueingRabbit.trigger_event(:consuming_done)
+          remove_pidfile
+        }
+      end
+
+      Signal.trap("TERM", &handler)
+      Signal.trap("INT", &handler)
+    end
+  end
+end