tchandy-minion 0.1.13

data/README.rdoc

@@ -0,0 +1,106 @@
+
+= Minion: super simple job queue over amqp
+
+Minion makes processing jobs over AMQP simple and easy.
+
+== Setup
+
+Minion pulls the AMQP credentials out the environment via AMQP_URL.
+
+	$ export AMQP_URL="amqp://johndoe:abc123@localhost/my_vhost"
+
+Alternativly you can explicitly set it programmatically like this:
+
+  Minion.amqp_url = "amqp://johndoe:abc123@localhost/my_vhost"
+
+If no URL is supplied, Minion defaults to "amqp://guest:guest@localhost/" which
+is the default credentials for Rabbitmq running locally.
+
+== Principles
+
+Minion treats your jobs with respect.  The queues are durable and not
+autodelete.  When popping jobs off the queue, they will not receive an ack
+until the job is done.  You can rest assured that once queued, the job will not
+be lost.
+
+Sends are done synchronously and receives are done asynchronously.  This allows
+you to Minion.enqueue() from the console, or in a mongrel and you don't need to
+worry about eventmachine.  It also means that when enqueue returns, the AMQP
+server has received your message. Daemons set to receive messages however use
+eventmachine.
+
+Message processing is done one at a time (prefetch 1).  If you want tasks done
+in parallel, run two minions.
+
+== Push a job onto the queue
+
+Its easy to push a job onto the queue.
+
+	Minion.enqueue("make.sandwich", { "for" => "me", "with" => "bread" })
+
+Minion expects a queue name (and will create it if needed). The second argument
+needs to be a hash.  
+
+== Processing a job
+
+	require 'minion'
+
+	include Minion
+
+	job "make.sandwich" do |args|
+		Sandwich.make(args["for"],args["with"])
+	end
+
+== Chaining multiple steps
+
+If you have a task that requires more than one step just pass an array of
+queues when you enqueue.
+
+	Minion.enqueue([ "make.sandwich", "eat.sandwich" ], "for" => "me")
+
+	job "make.sandwich" do
+		## this return value is merged with for => me and sent to the next queue
+		{ "type" => "ham on rye" }  
+	end
+
+	job "eat.sandwich" do |args|
+		puts "I have #{args["type"]} sandwich for #{args["me"]}"
+	end
+
+== Conditional Processing
+
+If you want a minion worker to only subscribe to a queue under specific
+conditions there is a :when parameter that takes a lambda as an argument.  For
+example, if you had a queue that makes sandwiches but only if there is bread
+on hand, it would be.
+
+  job "make.sandwich", :when => lambda { not Bread.out? } do
+    Sandwich.make
+  end
+
+== Error handling
+
+When an error is thrown in a job handler, the job is requeued to be done later
+and the minion process exits.  If you define an error handler, however, the
+error handler is run and the job is removed from the queue.
+
+	error do |e|
+		puts "got an error! #{e}"
+	end
+
+== Logging
+
+Minion logs to stdout via "puts".  You can specify a custom logger like this:
+
+	logger do |msg|
+		puts msg
+	end
+
+== Meta
+
+Created by Orion Henry
+
+Patches contributed by Adam Wiggins, Kyle Drake
+
+Released under the MIT License: www.opensource.org/licenses/mit-license.php
+

data/Rakefile

@@ -0,0 +1,24 @@
+require 'jeweler'
+
+Jeweler::Tasks.new do |s|
+  s.name = "tchandy-minion"
+  s.description = "Super simple job queue over AMQP"
+  s.summary = s.description
+  s.author = "Orion Henry, Thiago Pradi"
+  s.email = "orion@heroku.com, tchandy@gmail.com"
+  s.homepage = "http://github.com/tchandy/minion"
+  s.rubyforge_project = "minion"
+  s.files = FileList["[A-Z]*", "{bin,lib,spec}/**/*"]
+  s.add_dependency "amqp", ">= 0.6.6"
+  s.add_dependency "bunny", ">= 0.6.0"
+  s.add_dependency "json", ">= 1.2.0"
+end
+
+Jeweler::RubyforgeTasks.new
+
+desc 'Run specs'
+task :spec do
+  sh 'bacon -s spec/*_spec.rb'
+end
+
+task :default => :spec

data/VERSION

@@ -0,0 +1 @@
+0.1.13

data/examples/math.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'rubygems'
+require 'minion'
+
+include Minion
+
+error do |exception,queue,message,headers|
+	puts "got an error processing queue #{queue}"
+  puts exception.message
+  puts exception.backtrace
+end
+
+logger do |msg|
+	puts "--> #{msg}"
+end
+
+job "math.incr" do |args|
+	{ "number" => (1 + args["number"].to_i) }
+end
+
+job "math.double" do |args|
+	{ "number" => (2 * args["number"].to_i) }
+end
+
+job "math.square" do |args|
+	{ "number" => (args["number"].to_i * args["number"].to_i) }
+end
+
+job "math.print" do |args|
+	puts "NUMBER -----> #{args["number"]}"
+end
+
+enqueue([ "math.incr", "math.double", "math.square", "math.incr", "math.double", "math.print" ], { :number => 3 })
+

data/examples/sandwich.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'rubygems'
+require 'minion'
+
+include Minion
+
+error do |exception,queue,message,headers|
+  puts "got an error processing queue #{queue}"
+  puts exception.message
+  puts exception.backtrace
+end
+
+job "add.bread" do |args|
+	{ "bread" => "sourdough" }
+end
+
+job "add.meat" do |args|
+	{ "meat" => "turkey" }
+end
+
+job "add.condiments" do |args|
+	{ "condiments" => "mayo" }
+end
+
+job "eat.sandwich" do |args|
+	puts "YUM!  A #{args['meat']} on #{args['bread']} sandwich with #{args['condiments']}"
+end
+
+enqueue(["add.bread", "add.meat", "add.condiments", "eat.sandwich" ])
+

data/examples/when.rb

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'rubygems'
+require 'minion'
+
+include Minion
+
+error do |exception,queue,message,headers|
+  puts "got an error processing queue #{queue}"
+  puts exception.message
+  puts exception.backtrace
+end
+
+logger do |msg|
+	puts "--> #{msg}"
+end
+
+$listen = true
+
+job "do.once", :when => lambda { $listen }  do |args|
+	puts "Do this one action - then unsubscribe..."
+	$listen = false
+end
+
+enqueue("do.once",[])
+enqueue("do.once",[])
+

data/lib/minion.rb

@@ -0,0 +1,152 @@
+require 'uri'
+require 'json' unless defined? ActiveSupport::JSON
+require 'mq'
+require 'bunny'
+require 'minion/handler'
+
+module Minion
+  extend self
+
+  def url=(url)
+    @@config_url = url
+  end
+
+  def enqueue(jobs, data = {})
+    raise "cannot enqueue a nil job" if jobs.nil?
+    raise "cannot enqueue an empty job" if jobs.empty?
+
+    ## jobs can be one or more jobs
+    if jobs.respond_to? :shift
+      queue = jobs.shift
+      data["next_job"] = jobs unless jobs.empty?
+    else
+      queue = jobs
+    end
+
+    log "send: #{queue}:#{data.to_json}"
+    bunny.queue(queue, :durable => true, :auto_delete => false).publish(data.to_json)
+  end
+
+  def log(msg)
+    @@logger ||= proc { |m| puts "#{Time.now} :minion: #{m}" }
+    @@logger.call(msg)
+  end
+
+  def error(&blk)
+    @@error_handler = blk
+  end
+
+  def logger(&blk)
+    @@logger = blk
+  end
+
+  def job(queue, options = {}, &blk)
+    handler = Minion::Handler.new queue
+    handler.when = options[:when] if options[:when]
+    handler.unsub = lambda do
+      log "unsubscribing to #{queue}"
+      MQ.queue(queue).unsubscribe
+    end
+    handler.sub = lambda do
+      @@ack = true
+      log "subscribing to #{queue}"
+      MQ.queue(queue).subscribe(:ack => true) do |h,m|
+        return if AMQP.closing?
+        begin
+          log "recv: #{queue}:#{m}"
+
+          args = decode_json(m)
+
+          result = yield(args)
+
+          next_job(args, result)
+        rescue Object => e
+          raise unless error_handler
+          error_handler.call(e,queue,m,h)
+        end
+        h.ack if should_ack?
+        check_all
+      end
+    end
+    @@handlers ||= []
+    at_exit { Minion.run } if @@handlers.size == 0
+    @@handlers << handler
+  end
+
+  def decode_json(string)
+    if defined? ActiveSupport::JSON
+      ActiveSupport::JSON.decode string
+    else
+      JSON.load string
+    end
+  end
+
+  def check_all
+    @@handlers.each { |h| h.check }
+  end
+
+  def should_ack?
+    @@ack
+  end
+
+  def not_ack()
+    @@ack = false
+  end
+
+  def run
+    log "Starting minion"
+
+    Signal.trap('INT') { AMQP.stop{ EM.stop } }
+    Signal.trap('TERM'){ AMQP.stop{ EM.stop } }
+
+    EM.run do
+      AMQP.start(amqp_config) do
+        MQ.prefetch(1)
+        check_all
+      end
+    end
+  end
+
+  def amqp_url
+    @@amqp_url ||= ENV["AMQP_URL"] || "amqp://guest:guest@localhost/"
+  end
+
+  def amqp_url=(url)
+    @@amqp_url = url
+  end
+
+  private
+
+  def amqp_config
+    uri = URI.parse(amqp_url)
+    {
+      :vhost => uri.path,
+      :host => uri.host,
+      :user => uri.user,
+      :port => (uri.port || 5672),
+      :pass => uri.password
+    }
+  rescue
+    raise "invalid AMQP_URL: #{uri.inspect} (#{e})"
+  end
+
+  def new_bunny
+    b = Bunny.new(amqp_config)
+    b.start
+    b
+  end
+
+  def bunny
+    @@bunny ||= new_bunny
+  end
+
+  def next_job(args, response)
+    queue = args.delete("next_job")
+    enqueue(queue,args.merge(response)) if queue and not queue.empty?
+  end
+
+  def error_handler
+    @@error_handler ||= nil
+  end
+end
+

data/lib/minion/handler.rb

@@ -0,0 +1,30 @@
+module Minion
+	class Handler
+		attr_accessor :queue, :sub, :unsub, :when, :on
+		def initialize(queue)
+			@queue = queue
+			@when = lambda { true }
+			@sub = lambda {}
+			@unsub = lambda {}
+			@on = false
+		end
+
+		def should_sub?
+			@when.call
+		end
+
+		def check
+			if should_sub?
+				@sub.call unless @on
+				@on = true
+			else
+				@unsub.call if @on
+				@on = false
+			end
+		end
+
+		def to_s
+			"<handler queue=#{@queue} on=#{@on}>"
+		end
+	end
+end

data/spec/base.rb

@@ -0,0 +1,21 @@
+require File.dirname(__FILE__) + '/../lib/minion'
+
+require 'bacon'
+require 'mocha/standalone'
+require 'mocha/object'
+
+class Bacon::Context
+	include Mocha::API
+
+	def initialize(name, &block)
+		@name = name
+		@before, @after = [
+			[lambda { mocha_setup }],
+			[lambda { mocha_verify ; mocha_teardown }]
+		]
+		@block = block
+	end
+
+	def xit(desc, &bk)
+	end
+end

data/spec/enqueue_spec.rb

@@ -0,0 +1,10 @@
+require File.dirname(__FILE__) + '/base'
+
+describe Minion do
+	it "should throw an exception when passed a nil queue" do
+		lambda { Minion.enqueue(nil, {}) }.should.raise(RuntimeError)
+	end
+	it "should throw an exception when passed an empty queue" do
+		lambda { Minion.enqueue([], {}) }.should.raise(RuntimeError)
+  end
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification 
+name: tchandy-minion
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 13
+  version: 0.1.13
+platform: ruby
+authors: 
+- Orion Henry, Thiago Pradi
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-05-13 00:00:00 -03:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: amqp
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 6
+        - 6
+        version: 0.6.6
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: bunny
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 6
+        - 0
+        version: 0.6.0
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 2
+        - 0
+        version: 1.2.0
+  type: :runtime
+  version_requirements: *id003
+description: Super simple job queue over AMQP
+email: orion@heroku.com, tchandy@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/minion.rb
+- lib/minion/handler.rb
+- spec/base.rb
+- spec/enqueue_spec.rb
+has_rdoc: true
+homepage: http://github.com/tchandy/minion
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: minion
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Super simple job queue over AMQP
+test_files: 
+- spec/base.rb
+- spec/enqueue_spec.rb
+- examples/math.rb
+- examples/sandwich.rb
+- examples/when.rb