minion_kim 0.1.16

data/README.rdoc

@@ -0,0 +1,106 @@
+
+= Minion: super simple job queue over amqp
+
+Minion makes processing jobs over AMQP simple and easy.
+
+== Setup
+
+Minion pulls the AMQP credentials out the environment via AMQP_URL.
+
+	$ export AMQP_URL="amqp://johndoe:abc123@localhost/my_vhost"
+
+Alternativly you can explicitly set it programmatically like this:
+
+  Minion.amqp_url = "amqp://johndoe:abc123@localhost/my_vhost"
+
+If no URL is supplied, Minion defaults to "amqp://guest:guest@localhost/" which
+is the default credentials for Rabbitmq running locally.
+
+== Principles
+
+Minion treats your jobs with respect.  The queues are durable and not
+autodelete.  When popping jobs off the queue, they will not receive an ack
+until the job is done.  You can rest assured that once queued, the job will not
+be lost.
+
+Sends are done synchronously and receives are done asynchronously.  This allows
+you to Minion.enqueue() from the console, or in a mongrel and you don't need to
+worry about eventmachine.  It also means that when enqueue returns, the AMQP
+server has received your message. Daemons set to receive messages however use
+eventmachine.
+
+Message processing is done one at a time (prefetch 1).  If you want tasks done
+in parallel, run two minions.
+
+== Push a job onto the queue
+
+Its easy to push a job onto the queue.
+
+	Minion.enqueue("make.sandwich", { "for" => "me", "with" => "bread" })
+
+Minion expects a queue name (and will create it if needed). The second argument
+needs to be a hash.  
+
+== Processing a job
+
+	require 'minion'
+
+	include Minion
+
+	job "make.sandwich" do |args|
+		Sandwich.make(args["for"],args["with"])
+	end
+
+== Chaining multiple steps
+
+If you have a task that requires more than one step just pass an array of
+queues when you enqueue.
+
+	Minion.enqueue([ "make.sandwich", "eat.sandwich" ], "for" => "me")
+
+	job "make.sandwich" do
+		## this return value is merged with for => me and sent to the next queue
+		{ "type" => "ham on rye" }  
+	end
+
+	job "eat.sandwich" do |args|
+		puts "I have #{args["type"]} sandwich for #{args["me"]}"
+	end
+
+== Conditional Processing
+
+If you want a minion worker to only subscribe to a queue under specific
+conditions there is a :when parameter that takes a lambda as an argument.  For
+example, if you had a queue that makes sandwiches but only if there is bread
+on hand, it would be.
+
+  job "make.sandwich", :when => lambda { not Bread.out? } do
+    Sandwich.make
+  end
+
+== Error handling
+
+When an error is thrown in a job handler, the job is requeued to be done later
+and the minion process exits.  If you define an error handler, however, the
+error handler is run and the job is removed from the queue.
+
+	error do |e|
+		puts "got an error! #{e}"
+	end
+
+== Logging
+
+Minion logs to stdout via "puts".  You can specify a custom logger like this:
+
+	logger do |msg|
+		puts msg
+	end
+
+== Meta
+
+Created by Orion Henry
+
+Patches contributed by Adam Wiggins, Kyle Drake
+
+Released under the MIT License: www.opensource.org/licenses/mit-license.php
+

data/Rakefile

@@ -0,0 +1,21 @@
+require 'jeweler'
+
+Jeweler::Tasks.new do |s|
+  s.name = "minion_kim"
+  s.description = "Super simple job queue over AMQP"
+  s.summary = "Fork containing one simple patch. Will not be maintained if pull request #6 gets accepted upstream"
+  s.authors = ["Orion Henry", "Kim Altintop"]
+  s.email = "kim@soundcloud.com"
+  s.homepage = "http://github.com/kim/minion"
+  s.files = FileList["[A-Z]*", "{bin,lib,spec}/**/*"]
+  s.add_dependency "amqp", ">= 0.6.7"
+  s.add_dependency "bunny", ">= 0.6.0"
+  s.add_dependency "json", ">= 1.2.0"
+end
+
+desc 'Run specs'
+task :spec do
+  sh 'bacon -s spec/*_spec.rb'
+end
+
+task :default => :spec

data/VERSION

@@ -0,0 +1 @@
+0.1.16

data/lib/minion/handler.rb

@@ -0,0 +1,30 @@
+module Minion
+	class Handler
+		attr_accessor :queue, :sub, :unsub, :when, :on
+		def initialize(queue)
+			@queue = queue
+			@when = lambda { true }
+			@sub = lambda {}
+			@unsub = lambda {}
+			@on = false
+		end
+
+		def should_sub?
+			@when.call
+		end
+
+		def check
+			if should_sub?
+				@sub.call unless @on
+				@on = true
+			else
+				@unsub.call if @on
+				@on = false
+			end
+		end
+
+		def to_s
+			"<handler queue=#{@queue} on=#{@on}>"
+		end
+	end
+end

data/lib/minion.rb

@@ -0,0 +1,144 @@
+require 'uri'
+require 'json' unless defined? ActiveSupport::JSON
+require 'mq'
+require 'bunny'
+require 'minion/handler'
+
+module Minion
+	extend self
+
+	def url=(url)
+		@@config_url = url
+	end
+
+	def enqueue(jobs, data = {})
+		raise "cannot enqueue a nil job" if jobs.nil?
+		raise "cannot enqueue an empty job" if jobs.empty?
+
+		## jobs can be one or more jobs
+		if jobs.respond_to? :shift
+			queue = jobs.shift
+			data["next_job"] = jobs unless jobs.empty?
+		else
+			queue = jobs
+		end
+
+		encoded = JSON.dump(data)
+		log "send: #{queue}:#{encoded}"
+		bunny.queue(queue, :durable => true, :auto_delete => false).publish(encoded)
+	end
+
+	def log(msg)
+		@@logger ||= proc { |m| puts "#{Time.now} :minion: #{m}" }
+		@@logger.call(msg)
+	end
+
+	def error(&blk)
+		@@error_handler = blk
+	end
+
+	def logger(&blk)
+		@@logger = blk
+	end
+
+	def job(queue, options = {}, &blk)
+		handler = Minion::Handler.new queue
+		handler.when = options[:when] if options[:when]
+		handler.unsub = lambda do
+			log "unsubscribing to #{queue}"
+			MQ.queue(queue, :durable => true, :auto_delete => false).unsubscribe
+		end
+		handler.sub = lambda do
+			log "subscribing to #{queue}"
+			MQ.queue(queue, :durable => true, :auto_delete => false).subscribe(:ack => true) do |h,m|
+				return if AMQP.closing?
+				begin
+					log "recv: #{queue}:#{m}"
+
+					args = decode_json(m)
+
+					result = yield(args,h)
+
+					next_job(args, result)
+				rescue Object => e
+					raise unless error_handler
+					error_handler.call(e,queue,m,h)
+				end
+				h.ack
+				check_all
+			end
+		end
+		@@handlers ||= []
+		at_exit { Minion.run } if @@handlers.size == 0
+		@@handlers << handler
+	end
+
+	def decode_json(string)
+		if defined? ActiveSupport::JSON
+			ActiveSupport::JSON.decode string
+		else
+			JSON.load string
+		end
+	end
+
+	def check_all
+		@@handlers.each { |h| h.check }
+	end
+
+	def run
+		log "Starting minion"
+
+		Signal.trap('INT') { AMQP.stop{ EM.stop } }
+		Signal.trap('TERM'){ AMQP.stop{ EM.stop } }
+
+		EM.run do
+			AMQP.start(amqp_config) do
+				MQ.prefetch(1)
+				check_all
+			end
+		end
+	end
+
+	def amqp_url
+		@@amqp_url ||= ENV["AMQP_URL"] || "amqp://guest:guest@localhost/"
+	end
+
+	def amqp_url=(url)
+		@@amqp_url = url
+	end
+
+	private
+
+	def amqp_config
+		uri = URI.parse(amqp_url)
+		{
+			:vhost => uri.path,
+			:host => uri.host,
+			:user => uri.user,
+			:port => (uri.port || 5672),
+			:pass => uri.password
+		}
+	rescue Object => e
+		raise "invalid AMQP_URL: #{uri.inspect} (#{e})"
+	end
+
+	def new_bunny
+		b = Bunny.new(amqp_config)
+		b.start
+		b
+	end
+
+	def bunny
+		@@bunny ||= new_bunny
+	end
+
+	def next_job(args, response)
+		queue = args.delete("next_job")
+		enqueue(queue,args.merge(response)) if queue and not queue.empty?
+	end
+
+	def error_handler
+		@@error_handler ||= nil
+	end
+end
+

data/spec/base.rb

@@ -0,0 +1,21 @@
+require File.dirname(__FILE__) + '/../lib/minion'
+
+require 'bacon'
+require 'mocha/standalone'
+require 'mocha/object'
+
+class Bacon::Context
+	include Mocha::API
+
+	def initialize(name, &block)
+		@name = name
+		@before, @after = [
+			[lambda { mocha_setup }],
+			[lambda { mocha_verify ; mocha_teardown }]
+		]
+		@block = block
+	end
+
+	def xit(desc, &bk)
+	end
+end

data/spec/enqueue_spec.rb

@@ -0,0 +1,10 @@
+require File.dirname(__FILE__) + '/base'
+
+describe Minion do
+	it "should throw an exception when passed a nil queue" do
+		lambda { Minion.enqueue(nil, {}) }.should.raise(RuntimeError)
+	end
+	it "should throw an exception when passed an empty queue" do
+		lambda { Minion.enqueue([], {}) }.should.raise(RuntimeError)
+  end
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: minion_kim
+version: !ruby/object:Gem::Version
+  version: 0.1.16
+  prerelease: 
+platform: ruby
+authors:
+- Orion Henry
+- Kim Altintop
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-07-13 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: amqp
+  requirement: &2153999176 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.6.7
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153999176
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: &2153998504 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153998504
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &2153997832 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153997832
+description: Super simple job queue over AMQP
+email: kim@soundcloud.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.rdoc
+files:
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/minion.rb
+- lib/minion/handler.rb
+- spec/base.rb
+- spec/enqueue_spec.rb
+homepage: http://github.com/kim/minion
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.5
+signing_key: 
+specification_version: 3
+summary: ! 'Fork containing one simple patch. Will not be maintained if pull request
+  #6 gets accepted upstream'
+test_files: []