adamwiggins-minion 0.1.2

data/README.rdoc

@@ -0,0 +1,83 @@
+
+= Minion: super simple job queue over amqp
+
+Minion makes processing jobs over AMQP simple and easy.
+
+== Setup
+
+Minion pulls the AMQP credentials out the environment via AMQP_URI.
+
+	$ export AMQP_URI="amqp://johndoe:abc123@localhost/my_vhost"
+
+If no URI is supplied, Minion defaults to "amqp://guest:guest@localhost/" which
+is the default credentials for Rabbitmq running locally.
+
+== Principles
+
+Minion treats your jobs with respect.  The queues are durable and not
+autodelete.  When popping jobs off the queue, they will not receive an ack
+until the job is done.  You can rest assured that once queued, the job will not
+be lost.
+
+Sends are done synchronously and receives are done asynchronously.  This allows
+you to Minion.enqueue() from the console, or in a mongrel and you don't need to
+worry about eventmachine.  It also means that when enqueue returns, the AMQP
+server has received your message. Daemons set to receive messages however use
+eventmachine.
+
+Message processing is done one at a time (prefetch 1).  If you want tasks done
+in parallel, run two minions.
+
+== Push a job onto the queue
+
+Its easy to push a job onto the queue.
+
+	Minion.enqueue("make.sandwich", { "for" => "me", "with" => "bread" })
+
+Minion expects a queue name (and will create it if needed). The second argument
+needs to be a hash.  
+
+== Processing a job
+
+	require 'minion'
+
+	include Minion
+
+	job "make.sandwich" do |args|
+		Sandwich.make(args["for"],args["with"])
+	end
+
+== Chaining multiple steps
+
+If you have a task that requires more than one step just pass an array of
+queues when you enqueue.
+
+	Minion.enqueue([ "make.sandwich", "eat.sandwich" ], "for" => "me")
+
+	job "make.sandwich" do
+		## this return value is merged with for => me and sent to the next queue
+		{ "type" => "ham on rye" }  
+	end
+
+	job "eat.sandwich" do |args|
+		puts "I have #{args["type"]} sandwich for #{args["me"]}"
+	end
+
+== Error handling
+
+When an error is thrown in a job handler, the job is requeued to be done later
+and the minion process exits.  If you define an error handler, however, the
+error handler is run and the job is removed from the queue.
+
+	on_error do |e|
+		puts "got an error! #{e}"
+	end
+
+== Logging
+
+Minion logs to stdout via "puts".  You can specify a custom logger like this:
+
+	logger do |msg|
+		puts msg
+	end
+

data/VERSION

@@ -0,0 +1 @@
+0.1.2

data/examples/math.rb

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'minion'
+
+include Minion
+
+on_error do |e|
+	puts "got an error!"
+end
+
+logger do |msg|
+	puts "--> #{msg}"
+end
+
+job "math.incr" do |args|
+	{ "number" => (1 + args["number"].to_i) }
+end
+
+job "math.double" do |args|
+	{ "number" => (2 * args["number"].to_i) }
+end
+
+job "math.square" do |args|
+	{ "number" => (args["number"].to_i * args["number"].to_i) }
+end
+
+job "math.print" do |args|
+	puts "NUMBER -----> #{args["number"]}"
+end
+
+enqueue([ "math.incr", "math.double", "math.square", "math.incr", "math.double", "math.print" ], { :number => 3 })
+

data/examples/sandwich.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'minion'
+
+include Minion
+
+job "add.bread" do |args|
+	{ "bread" => "sourdough" }
+end
+
+job "add.meat" do |args|
+	{ "meat" => "turkey" }
+end
+
+job "add.condiments" do |args|
+	{ "condiments" => "mayo" }
+end
+
+job "eat.sandwich" do |args|
+	puts "YUM!  A #{args['meat']} on #{args['bread']} sandwich with #{args['condiments']}"
+end
+
+enqueue(["add.bread", "add.meat", "add.condiments", "eat.sandwich" ])
+

data/lib/minion.rb

@@ -0,0 +1,114 @@
+require 'uri'
+require 'json'
+require 'mq'
+require 'bunny'
+
+module Minion
+	extend self
+
+	def enqueue(jobs, data = {})
+		## jobs can be one or more jobs
+		if jobs.respond_to? :shift
+			queue = jobs.shift
+			data["next_job"] = jobs unless jobs.empty?
+		else
+			queue = jobs
+		end
+
+		log "send: #{queue}:#{data.to_json}"
+		bunny.queue(queue, :durable => true, :auto_delete => false).publish(data.to_json)
+	end
+
+	def on_error(&blk)
+		@@error_handler = blk
+	end
+
+	def logger(&blk)
+		@@logger = blk
+	end
+
+	def job(queue, &blk)
+		handler do
+			MQ.queue(queue).subscribe(:ack => true) do |h,m|
+				return if AMQP.closing?
+				begin
+					log "recv: #{queue}:#{m}"
+
+					args = JSON.load(m)
+
+					result = yield(args)
+
+					next_job(args, result)
+				rescue Object => e
+					raise unless error_handler
+					error_handler.call(e)
+				end
+				h.ack
+			end
+		end
+	end
+
+	def run
+		log "Starting minion"
+
+		Signal.trap('INT') { AMQP.stop{ EM.stop } }
+		Signal.trap('TERM'){ AMQP.stop{ EM.stop } }
+
+		EM.run do
+			AMQP.start(amqp_config) do
+				MQ.prefetch(1)
+				@@handlers.each { |h| h.call }
+			end
+		end
+	end
+
+	private
+
+	def amqp_url
+		ENV["AMQP_URL"] || "amqp://guest:guest@localhost/"
+	end
+
+	def amqp_config
+		uri = URI.parse(amqp_url)
+		{
+			:vhost => uri.path,
+			:host => uri.host,
+			:user => uri.user,
+			:port => (uri.port || 5672),
+			:pass => uri.password
+		}
+	rescue
+		raise "invalid AMQP_URL: #{uri.inspect} (#{e})"
+	end
+
+	def new_bunny
+		b = Bunny.new(amqp_config)
+		b.start
+		b
+	end
+
+	def bunny
+		@@bunny ||= new_bunny
+	end
+
+	def log(msg)
+		@@logger ||= proc { |m| puts "#{Time.now} :minion: #{m}" }
+		@@logger.call(msg)
+	end
+
+	def handler(&blk)
+		@@handlers ||= []
+		at_exit { Minion.run } if @@handlers.size == 0
+		@@handlers << blk
+	end
+
+	def next_job(args, response)
+		queue = args.delete("next_job")
+		enqueue(queue,args.merge(response)) if queue
+	end
+
+	def error_handler
+		@@error_handler ||= nil
+	end
+end
+

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification 
+name: adamwiggins-minion
+version: !ruby/object:Gem::Version 
+  version: 0.1.2
+platform: ruby
+authors: 
+- Orion Henry
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-08-19 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: tmm1-amqp
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.6.4
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: json
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.1.7
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: bunny
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.5.2
+    version: 
+description: Super simple job queue over AMQP
+email: orion@heroku.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- README.rdoc
+- VERSION
+- examples/math.rb
+- examples/sandwich.rb
+- lib/minion.rb
+has_rdoc: true
+homepage: http://github.com/orionz/minion
+licenses: 
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: minion
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 2
+summary: Super simple job queue over AMQP
+test_files: 
+- examples/math.rb
+- examples/sandwich.rb