af_minion 0.1.15.1

data/README.rdoc

@@ -0,0 +1,106 @@
+
+= Minion: super simple job queue over amqp
+
+Minion makes processing jobs over AMQP simple and easy.
+
+== Setup
+
+Minion pulls the AMQP credentials out the environment via AMQP_URL.
+
+	$ export AMQP_URL="amqp://johndoe:abc123@localhost/my_vhost"
+
+Alternativly you can explicitly set it programmatically like this:
+
+  Minion.amqp_url = "amqp://johndoe:abc123@localhost/my_vhost"
+
+If no URL is supplied, Minion defaults to "amqp://guest:guest@localhost/" which
+is the default credentials for Rabbitmq running locally.
+
+== Principles
+
+Minion treats your jobs with respect.  The queues are durable and not
+autodelete.  When popping jobs off the queue, they will not receive an ack
+until the job is done.  You can rest assured that once queued, the job will not
+be lost.
+
+Sends are done synchronously and receives are done asynchronously.  This allows
+you to Minion.enqueue() from the console, or in a mongrel and you don't need to
+worry about eventmachine.  It also means that when enqueue returns, the AMQP
+server has received your message. Daemons set to receive messages however use
+eventmachine.
+
+Message processing is done one at a time (prefetch 1).  If you want tasks done
+in parallel, run two minions.
+
+== Push a job onto the queue
+
+Its easy to push a job onto the queue.
+
+	Minion.enqueue("make.sandwich", { "for" => "me", "with" => "bread" })
+
+Minion expects a queue name (and will create it if needed). The second argument
+needs to be a hash.  
+
+== Processing a job
+
+	require 'minion'
+
+	include Minion
+
+	job "make.sandwich" do |args|
+		Sandwich.make(args["for"],args["with"])
+	end
+
+== Chaining multiple steps
+
+If you have a task that requires more than one step just pass an array of
+queues when you enqueue.
+
+	Minion.enqueue([ "make.sandwich", "eat.sandwich" ], "for" => "me")
+
+	job "make.sandwich" do
+		## this return value is merged with for => me and sent to the next queue
+		{ "type" => "ham on rye" }  
+	end
+
+	job "eat.sandwich" do |args|
+		puts "I have #{args["type"]} sandwich for #{args["me"]}"
+	end
+
+== Conditional Processing
+
+If you want a minion worker to only subscribe to a queue under specific
+conditions there is a :when parameter that takes a lambda as an argument.  For
+example, if you had a queue that makes sandwiches but only if there is bread
+on hand, it would be.
+
+  job "make.sandwich", :when => lambda { not Bread.out? } do
+    Sandwich.make
+  end
+
+== Error handling
+
+When an error is thrown in a job handler, the job is requeued to be done later
+and the minion process exits.  If you define an error handler, however, the
+error handler is run and the job is removed from the queue.
+
+	error do |e|
+		puts "got an error! #{e}"
+	end
+
+== Logging
+
+Minion logs to stdout via "puts".  You can specify a custom logger like this:
+
+	logger do |msg|
+		puts msg
+	end
+
+== Meta
+
+Created by Orion Henry
+
+Patches contributed by Adam Wiggins, Kyle Drake
+
+Released under the MIT License: www.opensource.org/licenses/mit-license.php
+

data/Rakefile

@@ -0,0 +1,24 @@
+require 'jeweler'
+
+Jeweler::Tasks.new do |s|
+  s.name = "minion"
+  s.description = "Super simple job queue over AMQP"
+  s.summary = s.description
+  s.author = "Orion Henry"
+  s.email = "orion@heroku.com"
+  s.homepage = "http://github.com/orionz/minion"
+  s.rubyforge_project = "minion"
+  s.files = FileList["[A-Z]*", "{bin,lib,spec}/**/*"]
+  s.add_dependency "amqp", ">= 0.6.7"
+  s.add_dependency "bunny", ">= 0.6.0"
+  s.add_dependency "json", ">= 1.2.0"
+end
+
+Jeweler::RubyforgeTasks.new
+
+desc 'Run specs'
+task :spec do
+  sh 'bacon -s spec/*_spec.rb'
+end
+
+task :default => :spec

data/VERSION

@@ -0,0 +1 @@
+0.1.15

data/examples/math.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'rubygems'
+require 'minion'
+
+include Minion
+
+error do |exception,queue,message,headers|
+	puts "got an error processing queue #{queue}"
+	puts exception.message
+	puts exception.backtrace
+end
+
+logger do |msg|
+	puts "--> #{msg}"
+end
+
+job "math.incr" do |args|
+	{ "number" => (1 + args["number"].to_i) }
+end
+
+job "math.double" do |args|
+	{ "number" => (2 * args["number"].to_i) }
+end
+
+job "math.square" do |args|
+	{ "number" => (args["number"].to_i * args["number"].to_i) }
+end
+
+job "math.print" do |args|
+	puts "NUMBER -----> #{args["number"]}"
+end
+
+enqueue([ "math.incr", "math.double", "math.square", "math.incr", "math.double", "math.print" ], { :number => 3 })
+

data/examples/sandwich.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'rubygems'
+require 'minion'
+
+include Minion
+
+error do |exception,queue,message,headers|
+	puts "got an error processing queue #{queue}"
+	puts exception.message
+	puts exception.backtrace
+end
+
+job "add.bread" do |args|
+	{ "bread" => "sourdough" }
+end
+
+job "add.meat" do |args|
+	{ "meat" => "turkey" }
+end
+
+job "add.condiments" do |args|
+	{ "condiments" => "mayo" }
+end
+
+job "eat.sandwich" do |args|
+	puts "YUM!	A #{args['meat']} on #{args['bread']} sandwich with #{args['condiments']}"
+end
+
+enqueue(["add.bread", "add.meat", "add.condiments", "eat.sandwich" ])
+

data/examples/when.rb

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'rubygems'
+require 'minion'
+
+include Minion
+
+error do |exception,queue,message,headers|
+	puts "got an error processing queue #{queue}"
+	puts exception.message
+	puts exception.backtrace
+end
+
+logger do |msg|
+	puts "--> #{msg}"
+end
+
+$listen = true
+
+job "do.once", :when => lambda { $listen } do |args|
+	puts "Do this one action - then unsubscribe..."
+	$listen = false
+end
+
+enqueue("do.once",[])
+enqueue("do.once",[])
+

data/lib/minion.rb

@@ -0,0 +1,214 @@
+require 'uri'
+require 'json' unless defined? ActiveSupport::JSON
+require 'mq'
+require 'bunny'
+require 'minion/handler'
+
+module Minion
+  
+  class AMQPServerConnectError < StandardError; end;
+  
+  NUMBER_OF_RETRIES = 3
+  
+	extend self
+
+	def url=(url)
+		@@config_url = url
+	end
+
+  def enqueue(jobs, data = {})
+    raise "cannot enqueue a nil job" if jobs.nil?
+    raise "cannot enqueue an empty job" if jobs.empty?
+
+    ## jobs can be one or more jobs
+    if jobs.respond_to? :shift
+      queue = jobs.shift
+      data["next_job"] = jobs unless jobs.empty?
+    else
+      queue = jobs
+    end
+
+    # NOTE: we encode to yaml, Minion encodes to json.  The reason is that json cannot handle Ruby symbols    
+    yamlized_data = data.to_yaml
+    log "send: #{queue}:#{yamlized_data}"
+
+    # NOTE: When rabbitMQ goes down, the mongrel keeps on holding the stale connection to it. We need to establish a new
+    # connection when this happens.
+    
+    with_bunny_connection_retry do
+      bunny.queue(queue, :durable => false, :auto_delete => false).publish(yamlized_data)
+    end
+  end
+
+	def log(msg)
+		@@logger ||= proc { |m| puts "#{Time.now} :minion: #{m}" }
+		@@logger.call(msg)
+	end
+
+	def error(&blk)
+		@@error_handler = blk
+	end
+
+	def logger(&blk)
+		@@logger = blk
+	end
+
+	def job(queue, options = {}, &blk)
+		handler = Minion::Handler.new queue
+		handler.when = options[:when] if options[:when]
+		handler.unsub = lambda do
+			log "unsubscribing to #{queue}"
+			MQ.queue(queue, :durable => false, :auto_delete => false).unsubscribe
+		end
+
+		handler.sub = lambda do
+			log "subscribing to #{queue}"
+			MQ.queue(queue, :durable => false, :auto_delete => false).subscribe(:ack => false) do |h,m|
+				return if AMQP.closing?
+				begin
+					log "recv: #{queue}:#{m}"
+
+					args = decode_json(m)
+
+					result = yield(args)
+
+					next_job(args, result)
+				rescue Object => e
+					raise unless error_handler
+					error_handler.call(e,queue,m,h)
+				end
+				check_all
+			end
+		end
+		@@handlers ||= []
+		at_exit { Minion.run } if @@handlers.size == 0
+		@@handlers << handler
+	end
+
+	def decode_json(string)
+		if defined? ActiveSupport::JSON
+			ActiveSupport::JSON.decode string
+		else
+			JSON.load string
+		end
+	end
+
+	def check_all
+		@@handlers.each { |h| h.check }
+	end
+
+  def run
+    log "Starting minion"
+
+    Signal.trap('INT') { exit_handler }
+    Signal.trap('TERM'){ exit_handler }
+
+    EM.run do
+      connection = AMQP.start(amqp_config) do
+        MQ.prefetch(1)
+        check_all
+      end
+      
+      # NOTE: By default, Minion gets into a wierd state when there is a connection error (i.e., the AMQP server is down).  
+      # In short, it continues running without any subscriptions. We chose to detect this case, and kill the worker that
+      # has invoke run. This will allow it to be restarted by the monitoring system, and hopefully by then, the AMQP
+      # server will be back up. 
+      if connection.error?
+        exit_handler
+        raise AMQPServerConnectError.new("Couldn't connect to RabbitMQ server at #{amqp_url}")
+      end
+    end
+  end
+
+	def amqp_url
+		@@amqp_url ||= ENV["AMQP_URL"] || "amqp://guest:guest@localhost/"
+	end
+
+  def amqp_url=(url)
+    clear_bunny
+    @@amqp_url = url
+  end
+  
+  def delete!(*queue_names)
+    EM.run do 
+      AMQP.start(amqp_config) do
+        queue_names.each do |queue_name|
+          MQ.queue(queue_name).delete
+          msg = "Minion: deleting '#{queue_name}' queue from the AMQP server."
+          ActiveRecord::Base::logger.info(msg)
+          log msg
+        end
+      end
+
+      AMQP.stop { EM.stop }
+    end
+
+  end
+
+	private
+
+	def amqp_config
+		uri = URI.parse(amqp_url)
+		{
+			:vhost => uri.path,
+			:host => uri.host,
+			:user => uri.user,
+			:port => (uri.port || 5672),
+			:pass => uri.password
+		}
+	rescue Object => e
+		raise "invalid AMQP_URL: #{uri.inspect} (#{e})"
+	end
+
+	def new_bunny
+		b = Bunny.new(amqp_config)
+		b.start
+		b
+	end
+
+	def bunny
+		@@bunny ||= new_bunny
+	end
+
+	def next_job(args, response)
+		queue = args.delete("next_job")
+		enqueue(queue,args.merge(response)) if queue and not queue.empty?
+	end
+
+	def error_handler
+		@@error_handler ||= nil
+	end
+	
+	def exit_handler
+    AMQP.stop { EM.stop } 
+  end
+
+  def with_bunny_connection_retry
+    num_executions = 1 
+    begin
+      yield
+    rescue Bunny::ServerDownError, Bunny::ConnectionError => e
+      clear_bunny
+      if num_executions < NUMBER_OF_RETRIES
+        log "Retry ##{num_executions}  : #{caller.first}"
+        num_executions += 1
+        sleep 0.5
+        retry
+      else
+        error = e.class.new(e.message + " (Retried #{NUMBER_OF_RETRIES} times.  Giving up.)")
+        error.set_backtrace(e.backtrace)
+        raise error
+      end
+    rescue => e
+      clear_bunny
+      raise e
+    end            
+  end
+
+  def clear_bunny
+    @@bunny = nil    
+  end
+  
+  
+end
+

data/lib/minion/handler.rb

@@ -0,0 +1,30 @@
+module Minion
+	class Handler
+		attr_accessor :queue, :sub, :unsub, :when, :on
+		def initialize(queue)
+			@queue = queue
+			@when = lambda { true }
+			@sub = lambda {}
+			@unsub = lambda {}
+			@on = false
+		end
+
+		def should_sub?
+			@when.call
+		end
+
+		def check
+			if should_sub?
+				@sub.call unless @on
+				@on = true
+			else
+				@unsub.call if @on
+				@on = false
+			end
+		end
+
+		def to_s
+			"<handler queue=#{@queue} on=#{@on}>"
+		end
+	end
+end

data/spec/base.rb

@@ -0,0 +1,21 @@
+require File.dirname(__FILE__) + '/../lib/minion'
+
+require 'bacon'
+require 'mocha/standalone'
+require 'mocha/object'
+
+class Bacon::Context
+	include Mocha::API
+
+	def initialize(name, &block)
+		@name = name
+		@before, @after = [
+			[lambda { mocha_setup }],
+			[lambda { mocha_verify ; mocha_teardown }]
+		]
+		@block = block
+	end
+
+	def xit(desc, &bk)
+	end
+end

data/spec/enqueue_spec.rb

@@ -0,0 +1,10 @@
+require File.dirname(__FILE__) + '/base'
+
+describe Minion do
+	it "should throw an exception when passed a nil queue" do
+		lambda { Minion.enqueue(nil, {}) }.should.raise(RuntimeError)
+	end
+	it "should throw an exception when passed an empty queue" do
+		lambda { Minion.enqueue([], {}) }.should.raise(RuntimeError)
+  end
+end

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification 
+name: af_minion
+version: !ruby/object:Gem::Version 
+  hash: 121
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 15
+  - 1
+  version: 0.1.15.1
+platform: ruby
+authors: 
+- Orion Henry
+- AppFolio Dev Team
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-08-24 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: amqp
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 9
+        segments: 
+        - 0
+        - 6
+        - 7
+        version: 0.6.7
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: bunny
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 0
+        - 6
+        - 0
+        version: 0.6.0
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 31
+        segments: 
+        - 1
+        - 2
+        - 0
+        version: 1.2.0
+  type: :runtime
+  version_requirements: *id003
+description: Super simple job queue over AMQP with modifications from AppFolio Inc.
+email: tushar.ranka@appfolio.com orion@heroku.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/minion.rb
+- lib/minion/handler.rb
+- spec/base.rb
+- spec/enqueue_spec.rb
+- examples/math.rb
+- examples/sandwich.rb
+- examples/when.rb
+has_rdoc: true
+homepage: http://github.com/orionz/minion
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: af_minion
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Super simple job queue over AMQP
+test_files: 
+- spec/base.rb
+- spec/enqueue_spec.rb
+- examples/math.rb
+- examples/sandwich.rb
+- examples/when.rb