stooge 0.1.0

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/.rvmrc

@@ -0,0 +1 @@
+rvm 1.9.3@stooge --create

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in stooge.gemspec
+gemspec

data/HISTORY.md

@@ -0,0 +1,10 @@
+
+### 0.1.0 (2012-03-09)
+
+* Implemented same basic feature set as Minion (i.e. work queues)
+* Added ability to easily test job handlers
+* Made both #enqueue and #job asynchronous
+* Changed from using Bunny to the using official AMQP gem
+* Improved error handling logic and logging
+* Handling of message content types (JSON is still default)
+* Added reconnect to broker on connection failure support

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Niklas Holmgren, Sutajio
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,80 @@
+Stooge
+======
+
+Stooge is a fully EventMachine enabled DSL for interacting with an AMQP broker. Heavily inspired by Minion (but fresher and completely asynchronous).
+
+Setup
+-----
+
+Assuming you already have an AMQP broker installed:
+
+    $ gem install stooge
+
+You can configure the address to the broker using the ```AMQP_URL``` environment variable, or programmatically like this:
+
+```ruby
+Stooge.amqp_url = 'amqp://johndoe:abc123@localhost/my_vhost'
+```
+
+The default if not specified is ```amqp://guest:guest@localhost/```.
+
+Example usage
+-------------
+
+To process a job add the following to a file called ```worker.rb``` and run it with ```ruby worker.rb```. Stooge will start an EventMachine loop that waits for AMQP messages and processes matching jobs until you send ```SIG_INT``` or ```SIG_TERM``` to it.
+
+```ruby
+require 'stooge'
+
+Stooge.job('example.puts') do |args|
+  puts args['message']
+end
+```
+
+To push a job onto a queue you call ```Stooge.enqueue``` with the name of the work queue and the data you want to process. The data needs to be JSON serializable and can be for example a hash.
+
+```ruby
+require 'stooge'
+
+EM.run do
+  Stooge.enqueue('example.puts', :message => 'Hello, world!')
+end
+```
+
+Error handling
+--------------
+
+When an error is thrown in a job handler, the job is requeued to be done later and the Stooge process exits. If you define an error handler, however, the error handler is run and the job is removed from the queue.
+
+```ruby
+Stooge.error do |e|
+  puts "got an error! #{e}"
+end
+```
+
+Logging
+-------
+
+Stooge logs to stdout via ```puts```. You can specify a custom logger like this:
+
+```ruby
+Stooge.logger do |msg|
+  puts msg
+end
+```
+
+Testing
+-------
+
+To test the business logic in your job handler you can use the ```Stooge.run_handler``` helper method:
+
+```ruby
+Stooge.run_handler('example.work', { :foo => 'bar' }).should == 42
+```
+
+The return value is the return value from executing the job handler block.
+
+Author
+------
+
+Stooge was created by Niklas Holmgren (niklas@sutajio.se) with help from Martin Bruse (@zond) and released under the MIT license. Stooge is very much inspired by Minion (created by Orion Henry), Resque (created by Chris Wanstrath) and Stalker (created by Adam Wiggins).

data/Rakefile

@@ -0,0 +1,11 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+$LOAD_PATH.unshift 'lib'
+
+task :default => [:spec]
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = '--color'
+  t.pattern = 'spec/**/*_spec.rb'
+end

data/lib/stooge.rb

@@ -0,0 +1,248 @@
+require 'uri'
+require 'multi_json'
+require 'amqp'
+require 'em-synchrony'
+require 'stooge/version'
+require 'stooge/handler'
+require 'stooge/work_queue'
+require 'stooge/worker'
+
+module Stooge
+  extend self
+  extend Stooge::WorkQueue
+
+  @@connection = nil
+  @@channel = nil
+  @@handlers = []
+  @@error_handler = Proc.new do |exception, handler, payload, metadata|
+    Stooge.log "#{handler.queue_name} failed: #{exception.inspect}"
+    raise exception
+  end
+
+  #
+  # Will return a URL to the AMQP broker to use. Will get this from the
+  # <code>ENV</code> variable <code>AMQP_URL</code> if present, or use the
+  # default which is "amqp://guest:guest@localhost/" (the same as the default
+  # for a local RabbitMQ install).
+  #
+  # @return [String] a URL to an AMQP broker.
+  #
+  def amqp_url
+    @@amqp_url ||= ENV["AMQP_URL"] || "amqp://guest:guest@localhost/"
+  end
+
+  #
+  # Set the URL to the AMQP broker to use. The format of the URL should be
+  # "amqp://username:password@hostname/vhost" (vhost and username/password is
+  # optional).
+  #
+  def amqp_url=(url)
+    @@amqp_url = url
+  end
+
+  #
+  # Log message using the Stooge logger.
+  #
+  # @param [String] msg the message to log.
+  #
+  def log(msg)
+    @@logger ||= proc { |m| puts "#{Time.now} :stooge: #{m}" }
+    @@logger.call(msg)
+  end
+
+  #
+  # Configure a custom logger for Stooge. The block gets yielded with the
+  # log message. You can do whatever you want with it. The default behaviour
+  # is to simply output to stdout.
+  #
+  # @param [Proc] block a {::Proc} to yield when a message needs to be logged.
+  # @yieldparam [String] msg the message to log.
+  #
+  def logger(&blk)
+    @@logger = blk
+  end
+
+  #
+  # Configure a global error handler for Stooge jobs. The block gets yielded
+  # when a job handler raises an exception. The default error handler simply
+  # logs the error and re-raises the exception. You can use this to for
+  # example re-queue a failed job or send email notifications when a job
+  # fails.
+  #
+  # If you don't raise an exception in the error handler the job will be
+  # acked with the broker and the broker will consider the job done and remove
+  # it from the queue. If you for some reason want to force the job to be
+  # acked even when you raise an error you can manually ack it before you
+  # raise the error, like this:
+  #
+  #   Stooge.error do |exception, handler, payload, metadata|
+  #     metadata.ack
+  #     raise exception
+  #   end
+  #
+  # @param [Proc] block a {::Proc} to yield when an error happens.
+  # @yieldparam [Exception] exception the exception object raised.
+  # @yieldparam [Stooge::Handler] handler the handler that failed.
+  # @yieldparam [Object] payload the message payload that was processed when
+  #   the handler failed.
+  # @yieldparam [Hash] metadata the message metadata (headers, etc.)
+  #
+  def error(&blk)
+    @@error_handler = blk
+  end
+
+  #
+  # The global error handler.
+  #
+  # @return [Proc] the error handler block.
+  # 
+  def error_handler
+    @@error_handler
+  end
+
+  #
+  # Start listening to for new jobs on all queues using the specified channel.
+  #
+  # @param [AMQP::Channel] channel an open AMQP channel
+  #
+  def start_handlers(channel)
+    @@handlers.each { |h| h.start(channel) }
+  end
+
+  #
+  # Add a new job handler. Used by {Stooge.job}.
+  # 
+  # @param [Stooge::Handler] handler a handler object
+  #
+  def add_handler(handler)
+    @@handlers << handler
+  end
+
+  #
+  # Are there any job handlers defined? Used by {Stooge::Worker} to check if
+  # it should start a worker process.
+  #
+  # @return [Boolean] true or false
+  #
+  def handlers?
+    @@handlers.empty? == false
+  end
+
+  #
+  # Execute a handler block without going through AMQP at all. This is a
+  # helper method for use in tests. It allows you to test the business logic
+  # in a handler block without having to mess with the details of how Stooge
+  # works internally.
+  #
+  # Examples
+  #
+  #   Stooge.run_handler('example.work', :foo => 'bar').should == 42
+  #
+  # @param [String] queue_name the name of the handler to run
+  # @param [Object] data message data to send to the handler as arguments
+  # @param [Hash] headers optional headers to send as second argument to the
+  #   handler block
+  #
+  # @return the return value of the handler block
+  #
+  def run_handler(queue_name, data, headers = {})
+    @@handlers.each do |handler|
+      if handler.queue_name == queue_name
+        return handler.block.call(data, headers)
+      end
+    end
+  end
+
+  #
+  # Will yield an open and ready connection.
+  #
+  # @param [Proc] block a {::Proc} to yield an open and ready connection to.
+  #
+  def with_connection(&block)
+    if @@connection.nil? || @@connection.status == :closed
+      @@connection = AMQP.connect(amqp_config)
+      @@connection.on_tcp_connection_loss do
+        Stooge.log "[network failure] trying to reconnect..."
+        @@connection.reconnect
+      end
+      @@connection.on_recovery do
+        Stooge.log "connection with broker recovered"
+      end
+      @@connection.on_error do |ch, connection_close|
+         raise connection_close.reply_text
+       end
+    end
+    @@connection.on_open do
+      yield @@connection
+    end
+  end
+
+  #
+  # Will yield an open and ready channel.
+  #
+  # @param [Proc] block a {::Proc} to yield an open and ready channel to.
+  #
+  def with_channel(&block)
+    with_connection do |connection|
+      if @@channel.nil? || @@channel.status == :closed
+        @@channel = AMQP::Channel.new(connection, AMQP::Channel.next_channel_id, channel_options)
+        @@channel.on_error do |ch, channel_close|
+          Stooge.log "channel-level exception: code = #{channel_close.reply_code}, message = #{channel_close.reply_text}"
+        end
+      end
+      @@channel.once_open do
+        yield @@channel
+      end
+    end
+  end
+
+  # Start a {::Stooge} worker that consumes messages from the AMQP broker.
+  def start!
+    EM.synchrony do
+      with_channel do |channel|
+        start_handlers(channel)
+      end
+    end
+  end
+
+  # Will stop and deactivate {::Stooge}.
+  def stop!
+    EM.next_tick do
+      with_channel do |channel|
+        channel.close
+      end
+      @@channel = nil
+      with_connection do |connection|
+        connection.close
+      end
+      @@connection = nil
+      EM.stop
+    end
+  end
+
+  private
+
+    # Will return the default options to use when creating channels.
+    def channel_options
+      @channel_options ||= {
+        :prefetch => 1,
+        :auto_recovery => true
+      }
+    end
+
+    # Will return the default options when connecting to the AMQP broker.
+    # Uses the URL from {#amqp_url} to construct these options.
+    def amqp_config
+      uri = URI.parse(amqp_url)
+      {
+        :vhost => uri.path,
+        :host => uri.host,
+        :user => uri.user,
+        :port => (uri.port || 5672),
+        :pass => uri.password
+      }
+    rescue Object => e
+      raise "invalid AMQP_URL: #{uri.inspect} (#{e})"
+    end
+
+end

data/lib/stooge/handler.rb

@@ -0,0 +1,52 @@
+module Stooge
+  class Handler
+
+    attr_accessor :queue_name, :queue_options, :block
+
+    #
+    # Create a new handler object.
+    #
+    # @param [String] queue the name of the queue that this handler will pick
+    #   jobs from.
+    # @param [Hash] options handler options.
+    # @option options [Hash] :queue_options Options to use when creating the
+    #   queue.
+    #
+    def initialize(queue, options = {})
+      @queue_name = queue
+      @queue_options = options[:queue_options] || {}
+      @options = options
+      @block = lambda {}
+    end
+
+    #
+    # Start subscribing to the queue that this handler corresponds to. When
+    # a message arive; parse it and call the handler block with the data.
+    #
+    # @param [AMQP::Channel] channel an open AMQP channel
+    #
+    def start(channel)
+      Stooge.log "starting handler for #{@queue_name}"
+      channel.queue(@queue_name, @queue_options) do |queue|
+        queue.subscribe(:ack => true) do |metadata, payload|
+          Stooge.log "recv: #{@queue_name}"
+          begin
+            case metadata.content_type
+            when 'application/json'
+              args = MultiJson.decode(payload)
+            else
+              args = payload
+            end
+            @block.call(args, metadata.headers)
+          rescue Object => e
+            if Stooge.error_handler
+              Stooge.error_handler.call(e,self,payload,metadata)
+            end
+          end
+          metadata.ack
+        end
+      end
+    end
+
+  end
+end

data/lib/stooge/version.rb

@@ -0,0 +1,3 @@
+module Stooge
+  VERSION = "0.1.0"
+end

data/lib/stooge/work_queue.rb

@@ -0,0 +1,74 @@
+module Stooge
+  module WorkQueue
+
+    #
+    # Push a job onto a named queue.
+    #
+    # Example:
+    #
+    #   Stooge.enqueue('example.work')
+    #
+    # @param [String] queue_name The name of a work queue or an array of
+    #   names that the job will be sent through in sequential order (a
+    #   workflow).
+    # @param [Object] data The data to send as input to the job. Needs to be
+    #   JSON serializable.
+    # @param [Hash] headers AMQP headers to include in the job that gets
+    #   pushed. Needs to be a hash with key/value pairs.
+    #
+    def enqueue(queue_name, data, headers = {})
+      EM::Synchrony.sync(aenqueue(queue_name, data, headers))
+    end
+
+    #
+    # Asynchrounous version of enqueue. Yields when the job has been put onto
+    # the queue, if a block is given.
+    #
+    # @param [String] queue_name The name of a work queue or an array of
+    #   names that the job will be sent through in sequential order (a
+    #   workflow).
+    # @param [Object] data The data to send as input to the job. Needs to be
+    #   JSON serializable.
+    # @param [Hash] headers AMQP headers to include in the job that gets
+    #   pushed. Needs to be a hash with key/value pairs.
+    #
+    # @return [EM::DefaultDeferrable] Returns an EM::DefaultDeferrable object.
+    # 
+    def aenqueue(queue_name, data, headers = {})
+      deferrable = EM::DefaultDeferrable.new
+      with_channel do |channel|
+        options = {
+          :routing_key => queue_name,
+          :mandatory => true,
+          :content_type => 'application/json',
+          :headers => headers
+        }
+        channel.default_exchange.publish(MultiJson.encode(data), options) do
+          Stooge.log("enqueue: #{queue_name}(#{data})")
+          yield if block_given?
+          deferrable.set_deferred_status :succeeded
+        end
+      end
+      deferrable
+    end
+
+    #
+    # Creates a job handler for a named queue.
+    #
+    # Example:
+    #
+    #   Stooge.job('example.work') do |args,headers|
+    #     # Do the work here...
+    #   end
+    #
+    # @param [String] queue The name of the work queue.
+    # @param [Proc] blk a {::Proc} that processes the jobs.
+    #
+    def job(queue, &blk)
+      handler = Stooge::Handler.new(queue, :queue_options => { :durable => true })
+      handler.block = blk
+      add_handler(handler)
+    end
+
+  end
+end

data/lib/stooge/worker.rb

@@ -0,0 +1,61 @@
+module Stooge
+  class Worker
+
+    #
+    # Start the Stooge worker that processes jobs.
+    #
+    def self.run!
+      Stooge.log "Starting stooge"
+
+      Signal.trap('INT') { Stooge.stop! }
+      Signal.trap('TERM'){ Stooge.stop! }
+
+      Stooge.start!
+    end
+
+    #
+    # Should the Stooge worker be started?
+    #
+    # @return [Boolean] true or false
+    #
+    def self.run?
+      Stooge.handlers? &&
+        File.expand_path($0) == File.expand_path(app_file)
+    end
+
+    private
+
+      CALLERS_TO_IGNORE = [
+        /\/stooge(\/worker)?\.rb$/,
+        /rubygems\/custom_require\.rb$/,
+        /bundler(\/runtime)?\.rb/,
+        /<internal:/
+      ]
+
+      CALLERS_TO_IGNORE.concat(RUBY_IGNORE_CALLERS) if defined?(RUBY_IGNORE_CALLERS)
+
+      def self.caller_files
+        cleaned_caller(1).flatten
+      end
+
+      def self.caller_locations
+        cleaned_caller 2
+      end
+
+      def self.cleaned_caller(keep = 3)
+        caller(1).
+          map    { |line| line.split(/:(?=\d|in )/, 3)[0,keep] }.
+          reject { |file, *_| CALLERS_TO_IGNORE.any? { |pattern| file =~ pattern } }
+      end
+
+      def self.app_file
+        caller_files.first || $0
+      end
+
+  end
+
+  #
+  # Starte the worker at exit, if appropriate.
+  #
+  at_exit { Worker.run! if $!.nil? && Worker.run? }
+end

data/spec/stooge_spec.rb

@@ -0,0 +1,9 @@
+dir = File.dirname(File.expand_path(__FILE__))
+$LOAD_PATH.unshift dir + '/../lib'
+
+require 'stooge'
+require 'rspec'
+
+describe Stooge do
+
+end

data/stooge.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "stooge/version"
+
+Gem::Specification.new do |s|
+  s.name        = "stooge"
+  s.version     = Stooge::VERSION
+  s.authors     = ["Niklas Holmgren"]
+  s.email       = ["niklas@sutajio.se"]
+  s.homepage    = "https://github.com/sutajio/stooge"
+  s.summary     = %q{Super advanced job queue over AMQP}
+  s.description = %q{Super advanced job queue over AMQP}
+
+  s.rubyforge_project = "stooge"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'amqp'
+  s.add_dependency 'em-synchrony'
+  s.add_dependency 'multi_json'
+  s.add_development_dependency 'rspec'
+
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: stooge
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Niklas Holmgren
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: amqp
+  requirement: &70330131821100 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70330131821100
+- !ruby/object:Gem::Dependency
+  name: em-synchrony
+  requirement: &70330131820660 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70330131820660
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: &70330131820200 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70330131820200
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70330131819740 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70330131819740
+description: Super advanced job queue over AMQP
+email:
+- niklas@sutajio.se
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rvmrc
+- Gemfile
+- HISTORY.md
+- LICENSE
+- README.md
+- Rakefile
+- lib/stooge.rb
+- lib/stooge/handler.rb
+- lib/stooge/version.rb
+- lib/stooge/work_queue.rb
+- lib/stooge/worker.rb
+- spec/stooge_spec.rb
+- stooge.gemspec
+homepage: https://github.com/sutajio/stooge
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: stooge
+rubygems_version: 1.8.17
+signing_key: 
+specification_version: 3
+summary: Super advanced job queue over AMQP
+test_files:
+- spec/stooge_spec.rb