mb-minion 0.2.0

data/Rakefile

@@ -0,0 +1,30 @@
+require "bundler"
+Bundler.setup
+
+require "rake"
+require "rake/rdoctask"
+require "rspec"
+require "rspec/core/rake_task"
+
+$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
+require "mb-minion/version"
+
+task :build do
+  system "gem build mb-minion.gemspec"
+end
+
+task :install => :build do
+  system "sudo gem install mb-minion-#{Minion::VERSION}.gem"
+end
+
+task :release => :build do
+  system "git tag -a #{Minion::VERSION} -m 'Tagging #{Minion::VERSION}'"
+  system "git push --tags"
+  system "gem push mb-minion-#{Minion::VERSION}.gem"
+end
+
+Rspec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = "spec/**/*_spec.rb"
+end
+
+task :default => :spec

data/examples/batch.rb

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+#
+# This example illustrates the use of batching
+# messages together.  This way work can be
+# distributed in small chunks, but worked on
+# in larger groups
+#
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'rubygems'
+require 'mb-minion'
+
+include Minion
+
+error do |exception,queue,message,headers|
+	puts "got an error processing queue #{queue}"
+	puts exception.message
+	puts exception.backtrace
+end
+
+logger do |msg|
+	puts "--> #{msg}"
+end
+
+
+job "do.many", :batch_size => 10 do |msg|
+	puts "got #{msg.batch.size} messages"
+end
+
+27.times{ Minion.enqueue 'do.many', {"something" => true} }

data/examples/batch_wait.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+#
+# This example illustrates the use batching
+# when we want to wait until the hit an exact
+# number of messages before running
+#
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'rubygems'
+require 'mb-minion'
+
+include Minion
+
+error do |exception,queue,message,headers|
+	puts "got an error processing queue #{queue}"
+	puts exception.message
+	puts exception.backtrace
+end
+
+logger do |msg|
+	puts "--> #{msg}"
+end
+
+
+job "do.only_in_tens", :batch_size => 10, :wait => true do |msg|
+	puts "got #{msg.batch.size} messages"
+end
+
+# We won't run this, but just so you know, this is the
+# same result as the "batch.rb" example 
+job "do.in_tens_or_emtpy", :batch_size => 10, :wait => false do |msg|
+	puts "got #{msg.batch.size} messages"
+end
+
+27.times{ Minion.enqueue 'do.only_in_tens', {"something" => true} }

data/examples/batch_wait_2.rb

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+#
+# This example illustrates the use batching
+# when we want to wait some # of seconds for an
+# exact number of messages before running,
+# but if we don't get that many, we'll go ahead
+# anyways.
+#
+# This prevents some odd batch counts, but allows
+# for flexibility of the queue size
+#
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'rubygems'
+require 'mb-minion'
+
+include Minion
+
+error do |exception,queue,message,headers|
+	puts "got an error processing queue #{queue}"
+	puts exception.message
+	puts exception.backtrace
+end
+
+logger do |msg|
+	puts "--> #{msg}"
+end
+
+Thread.new do
+  puts "------------------------------------------"
+  puts "First, no waiting with artificial delays"
+  puts "------------------------------------------"
+  3.times{ Minion.enqueue 'do.no_waiting', {"something" => true} }
+  sleep 0.2
+  4.times{ Minion.enqueue 'do.no_waiting', {"something" => true} }
+  sleep 5
+  puts "------------------------------------------"
+  puts "Now if we give it a second, all is well"
+  puts "------------------------------------------"
+  3.times{ Minion.enqueue 'do.in_tens_or_wait_2', {"something" => true} }
+  sleep 0.2
+  4.times{ Minion.enqueue 'do.in_tens_or_wait_2', {"something" => true} }
+end
+
+job "do.no_waiting", :batch_size => 10 do |msg|
+  puts "got #{msg.batch.size} messages"
+end
+
+job "do.in_tens_or_wait_2", :batch_size => 10, :wait => 2 do |msg|
+  puts "got #{msg.batch.size} messages"
+end

data/examples/math.rb

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+
+#
+# This example illustrates the use of chaining
+# callbacks to create building blocks with the
+# message content being just an integer
+#
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'rubygems'
+require 'minion'
+
+include Minion
+
+error do |exception,queue,message,headers|
+	puts "got an error processing queue #{queue}"
+	puts exception.message
+	puts exception.backtrace
+end
+
+logger do |msg|
+	puts "--> #{msg}"
+end
+
+job "math.incr" do |msg|
+	msg.content.to_i + 1
+end
+
+job "math.double" do |msg|
+	msg.content.to_i * 2
+end
+
+job "math.square" do |msg|
+	msg.content.to_i * msg.content.to_i
+end
+
+job "math.print" do |msg|
+	puts "NUMBER -----> #{msg.content}"
+end
+
+enqueue([ "math.incr", "math.double", "math.square", "math.incr", "math.double", "math.print" ], 3)
+

data/examples/sandwich.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+
+#
+# This example illustrates the use of chaining
+# callbacks to create building blocks with the
+# message content being just a hash
+#
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'rubygems'
+require 'mb-minion'
+
+include Minion
+
+error do |exception,queue,message,headers|
+	puts "got an error processing queue #{queue}"
+	puts exception.message
+	puts exception.backtrace
+end
+
+job "add.bread" do |msg|
+	msg.content.merge("bread" => "sourdough")
+end
+
+job "add.meat" do |msg|
+	msg.content.merge("meat" => "turkey")
+end
+
+job "add.condiments" do |msg|
+	msg.content.merge("condiments" => "mayo")
+end
+
+job "eat.sandwich" do |msg|
+	puts "YUM!	A #{msg['meat']} on #{msg['bread']} sandwich with #{msg['condiments']}"
+end
+
+enqueue(["add.bread", "add.meat", "add.condiments", "eat.sandwich" ])
+

data/examples/sandwich_batch.rb

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+
+#
+# This example illustrates the use batching
+# and chaining of callbacks in a single example.
+# Another example of doing a map-reduce operation.
+#
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'rubygems'
+require 'mb-minion'
+
+include Minion
+
+error do |exception,queue,message,headers|
+	puts "got an error processing queue #{queue}"
+	puts exception.message
+	puts exception.backtrace
+end
+
+MAKINGS = {
+  'bread'      => %w[wheat rye sourdough white pumpernickle],
+  'meat'       => %w[turkey ham pastrami salami],
+  'condiments' => %w[mayo mustard relish sourkraut]
+}
+
+job "add.bread", :batch_size => 5 do |msg|
+  puts "Puts making #{msg.batch.size} sandwiches"
+	msg.batch.map{|s| s.merge("bread" => MAKINGS['bread'].sample)}
+end
+
+job "add.meat" do |msg|
+	msg.map{|s| s.merge("meat" => MAKINGS['meat'].sample)}
+end
+
+job "add.condiments" do |msg|
+  msg.map{|s| s.merge("condiments" => MAKINGS['condiments'].sample)}
+end
+
+job "eat.sandwich" do |msg|
+  msg.each_with_index{|s, i| puts "SANDWICH ##{i}:  A #{s['meat']} on #{s['bread']} sandwich with #{s['condiments']}"}
+end
+
+15.times{ enqueue(["add.bread", "add.meat", "add.condiments", "eat.sandwich" ]) }

data/examples/when.rb

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+
+#
+# This example illustrates the use of the :when
+# parameter to allow/disallow subscription to the
+# queue
+#
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'rubygems'
+require 'mb-minion'
+
+include Minion
+
+error do |exception,queue,message,headers|
+	puts "got an error processing queue #{queue}"
+	puts exception.message
+	puts exception.backtrace
+end
+
+logger do |msg|
+	puts "--> #{msg}"
+end
+
+$listen = true
+
+job "do.once", :when => lambda { $listen } do |args|
+	puts "Do this one action - then unsubscribe..."
+	$listen = false
+end
+
+enqueue("do.once",[])
+enqueue("do.once",[])
+

data/lib/mb-ext/string.rb

@@ -0,0 +1,7 @@
+class String
+  # Fake out "force_encoding" if we're on 1.8
+  #
+  def force_encoding enc
+    self
+  end unless method_defined? :force_encoding
+end

data/lib/mb-minion.rb

@@ -0,0 +1,231 @@
+# encoding: utf-8
+require "amqp"
+require "bunny"
+require "json" unless defined? ActiveSupport::JSON
+require "uri"
+require "mb-minion/handler"
+require "mb-minion/version"
+require "mb-minion/message"
+require "mb-ext/string"
+
+module Minion
+  extend self
+
+  # Handle when an error gets raised.
+  #
+  # @example Handle the error.
+  #   Minion.error(exception)
+  #
+  # @param [ Exception ] exception The error that was raised.
+  #
+  # @return [ Object ] The output og the error handler block.
+  def alert(exception)
+    raise(exception) unless error_handling
+    error_handling.call(exception)
+  end
+
+  # Gets the hash of configuration options.
+  #
+  # @example Get the configuration hash.
+  #   Minion.config
+  #
+  # @return [ Hash ] The configuration options.
+  def config
+    uri = URI.parse(url)
+    {
+      :vhost => uri.path,
+      :host => uri.host,
+      :user => uri.user,
+      :port => (uri.port || 5672),
+      :pass => uri.password
+    }
+  rescue Object => e
+    raise("invalid AMQP_URL: #{uri.inspect} (#{e})")
+  end
+
+  # Add content to the supplied queue or queues. The hash will get converted to
+  # JSON and placed on the queue as the JSON string.
+  #
+  # @example Place data on a single queue.
+  #   Minion.enqueue("queue.name", { field: "value" })
+  #
+  # @example Place data on multiple queues.
+  #   Minion.enqueue([ "queue.first", "queue.second" ], { field: "value" })
+  #
+  # @param [ String, Array<String> ] name The name or names of the queues.
+  # @param [ Hash ] data The payload to send.
+  #
+  # @raise [ RuntimeError ] If the name is nil or empty.
+  def enqueue(queues, data = {})
+    raise "Cannot enqueue an empty or nil name" if queues.nil? || queues.empty?
+    # Wrap raw data when we receive it
+    data = {'content' => data} unless data.class == Hash && data['content']
+    if queues.respond_to? :shift
+      queue = queues.shift
+      data['callbacks'] = queues
+    else
+      queue = queues
+    end
+    
+    # @todo: Durran: Any multi-byte character in the JSON causes a bad_payload
+    #   error on the rabbitmq side. It seems a fix in the old amqp gem
+    #   regressed in the new fork.
+    encoded = JSON.dump(data).force_encoding("ISO-8859-1")
+    
+    Minion.info("Send: #{queue}:#{encoded}")
+    connect do |bunny|
+      q = bunny.queue(queue, :durable => true, :auto_delete => false)
+      e = bunny.exchange('') # Connect to default exchange
+      e.publish(encoded, :key => q.name) 
+    end
+  end
+    
+  # Get the message count for a specific queue
+  #
+  # @example Get the message count for queue 'minion.test'.
+  #   Minion.message_count('minion.test')
+  #
+  # @return [ Fixnum ] the number of messages
+  def message_count(queue)
+    connect do |bunny|
+      return bunny.queue(queue, :durable => true, :auto_delete => false).message_count
+    end
+  end
+
+  # Define an optional method of changing the ways errors get handled.
+  #
+  # @example Define a custom error handler.
+  #   Minion.error do |e|
+  #     puts "I got an error - #{e.message}"
+  #   end
+  #
+  # @param [ Proc ] block The block that will handle the error.
+  def error(&block)
+    @@error_handling = block
+  end
+
+  # Runs each of the handlers.
+  #
+  # @example Check all handlers.
+  #   Minion.check_handlers
+  def execute_handlers
+    @@handlers.each { |handler| handler.execute }
+  end
+
+  # Log the supplied information message.
+  #
+  # @example Log the message.
+  #   Minion.info("something happened")
+  #
+  # @return [ Object ] The output of the logging block.
+  def info(message)
+    logging.call(message)
+  end
+
+  # Sets up a subscriber to a queue to process jobs.
+  #
+  # @example Set up the subscriber.
+  #   Minion.job "my.queue.name" do |attributes|
+  #     puts "Here's the message data: #{attributes"
+  #   end
+  #
+  # @param [ String ] queue The queue to subscribe to.
+  # @param [ Hash ] options Options for the subscriber.
+  #
+  # @option options [ lambda ] :when Conditionally process the job.
+  # @option options [ boolean ] :ack Should we automatically ack the message?
+  def job(queue, options = {}, &block)
+    Minion::Handler.new(queue, block, options).tap do |handler|
+      @@handlers ||= []
+      at_exit { Minion.run } if @@handlers.size == 0
+      @@handlers << handler
+    end
+  end
+
+  # Define an optional method of changing the ways logging is handled.
+  #
+  # @example Define a custom logger.
+  #   Minion.logger do |message|
+  #     puts "Something did something - #{message}"
+  #   end
+  #
+  # @param [ Proc ] block The block that will handle the logging.
+  def logger(&block)
+    @@logging = block
+  end
+
+  # Runs the minion subscribers.
+  #
+  # @example Run the subscribers.
+  #   Minion.run
+  def run
+    Minion.info("Starting minion")
+    Signal.trap("INT") { AMQP.stop { EM.stop } }
+    Signal.trap("TERM") { AMQP.stop { EM.stop } }
+
+    EM.run do
+      AMQP.start(config) do
+        AMQP::Channel.new.prefetch(1)
+        execute_handlers
+      end
+    end
+  end
+
+  # Get the url for the amqp server.
+  #
+  # @example Get the url.
+  #   Minion.url
+  #
+  # @return [ String ] The url.
+  def url
+    @@url ||= (ENV["AMQP_URL"] || "amqp://guest:guest@localhost/")
+  end
+
+  # Set the url to the amqp server.
+  #
+  # @example Set the url.
+  #   Minion.url = "amqp://user:password@host:port/vhost"
+  #
+  # @return [ String ] The new url.
+  def url=(url)
+    @@url = url
+  end
+  
+  private
+
+  # Get the bunny instance which is used for the synchronous communication.
+  #
+  # @example Get the bunny.
+  #   Minion.bunny
+  #
+  # @return [ Bunny ] The new bunny, all configured.
+  def connect
+    Bunny.new(config).tap do |bunny|
+      bunny.start
+      yield(bunny) if block_given?
+      bunny.stop
+    end
+  end
+
+  # Get the error handler for this class.
+  #
+  # @example Get the error handler.
+  #   Minion.error_handling
+  #
+  # @return [ lambda, nil ] The handler or nil.
+  def error_handling
+    @@error_handling ||= nil
+  end
+
+  # Get the logger for this class. If nothing had been specified will default
+  # to a basic time/message print.
+  #
+  # @example Get the logger.
+  #   Minion.logging
+  #
+  # @return [ lambda ] The logger.
+  def logging
+    @@logging ||= lambda { |msg| puts("#{Time.now} :minion: #{msg}") }
+  end
+end
+