mb-minion 0.2.0

data/lib/mb-minion/handler.rb

@@ -0,0 +1,177 @@
+require 'pp'
+# encoding: utf-8
+module Minion
+  class Handler
+    attr_reader :queue_name, :block, :batch_size, :wait
+
+    # Executes the handler. Will subscribe to a queue or unsubscribe to it
+    # depending on the conditions.
+    #
+    # @example Execute the handler.
+    #   handler.execute
+    def execute
+      subscribable? ? subscribe : unsubscribe
+    end
+
+    # Instantiate the new handler. Takes a queue name and optional lambda to
+    # determine conditionally if a queue is subscribable.
+    #
+    # @example Create the new handler.
+    #   Handler.new("minion.test")
+    #
+    # @param [ String ] queue_name The name of the queue.
+    # @param [ Hash ] 
+    # @option options [ lambda ] :when The block for conditionally subscribing.
+    # @option options [ fixnum ] :batch_size The number of elements per batch
+    # @option options [ symbol ] :map The type of map operation: fanout or reduce
+    def initialize(queue_name, block, options = {})
+      @queue_name, @block = queue_name, block
+      @subscribable = options[:when]
+      @batch_size = options[:batch_size]
+      @wait = options[:wait] || false
+      raise ArgumentError, "wait parameter makes no sense without a batch_size" if (@wait && ! @batch_size)
+    end
+
+    private
+
+    # Returns true if the handler is already subscribed to the queue.
+    #
+    # @example Is the handler running?
+    #   handler.running?
+    #
+    # @return [ true, false ] Is the handler running?
+    def running?
+      !!@running
+    end
+
+    # Determines if the queue is able to be subscribed to.
+    #
+    # @example Is the queue subscribable?
+    #   handler.subscribable?
+    def subscribable?
+      @subscribable ? @subscribable.call : true
+    end
+
+    # Subscribe to the queue. Will do so if the handler is not already
+    # subscribed.
+    #
+    # @example Subscribe to the queue.
+    #   handler.subscribe
+    def subscribe
+      unless running?
+        Minion.info("Subscribing to #{queue_name}")
+        chan = AMQP::Channel.new
+        chan.prefetch(1)
+        queue = chan.queue(queue_name, :durable => true, :auto_delete => false)
+        if batch_size && batch_size > 1
+          process_batch(queue)
+        else
+          process_single_message(queue)
+        end
+        @running = true
+      end
+      
+    end
+    
+    # Process a multiple messages from a queue as a batch
+    #
+    # @example Subscribe to the queue.
+    #   handler.process_batch(queue)
+    # 
+    # @param [ AMQP::Queue ]
+    #
+    def process_batch(queue)
+      # Our batch message will have an array for it's content
+      msg = Message.new
+      queue.subscribe(:ack => true) do |h, m|
+        return if AMQP.closing?
+        Minion.info("Received: #{queue_name}:#{m}, #{h}")
+        args = decode(m)
+        
+        # All messages in the batch get the callbacks from
+        # the first message.  This is why when using chained
+        # callbacks on batches, you always have to use the
+        # same combo of callback-queues!
+        msg.callbacks = args['callbacks']
+        msg.batch << args['content']
+        h.ack # acks are useless in batch-mode.
+              # You'll have to make sure you requeue manually 
+        if (msg.batch.size == batch_size) || process_anyway?
+          msg.content = block.call(msg)
+          msg.callback
+          msg.batch.clear
+        end
+        Minion.execute_handlers
+      end
+    rescue Object => e
+      Minion.alert(e)
+    end
+    
+    # Process a single message from a queue
+    #
+    # @example Subscribe to the queue.
+    #   handler.process_single_message(queue)
+    # 
+    # @param [ AMQP::Queue ]
+    #
+    def process_single_message(queue)
+      queue.subscribe(:ack => true) do |h, m|
+        return if AMQP.closing?
+        Minion.info("Received: #{queue_name}:#{m}, #{h}")
+        msg = Message.new(m, h)
+        msg.content = block.call(msg)
+        h.ack
+        msg.callback
+        Minion.execute_handlers
+      end
+    rescue Object => e
+      Minion.alert(e)
+    end
+
+    # Get a string respresentation of the handler.
+    #
+    # @example Print out the string.
+    #   handler.to_s
+    #
+    # @return [ String ] The handler as a string.
+    def to_s
+      "<handler queue_name=#{@queue_name} on=#{@on}>"
+    end
+
+    # Unsubscribe from the queue.
+    #
+    # @example Unsubscribe from the queue.
+    #   handler.unsubscribe
+    def unsubscribe
+      Minion.info("Unsubscribing to #{queue_name}")
+      AMQP::Channel.new.queue(queue_name, :durable => true, :auto_delete => false).unsubscribe
+      @running = false
+    end
+    
+    private
+
+    def decode(json)
+      defined?(ActiveSupport::JSON) ?
+        ActiveSupport::JSON.decode(json) : JSON.load(json)
+    end
+    
+    # Determine if we should process a batch even if
+    # we haven't reached the batch_size
+    #
+    # @return [ Boolean ] if we should go ahead and process the batch
+    def process_anyway?
+      return false if Minion.message_count(queue_name) != 0 # there's work to be done!
+      case wait
+      when true  then false # Wait indefinitely
+      when false then true  # Don't wait at all
+      when Numeric
+        (0..wait).each do |i|
+          return false if Minion.message_count(queue_name) != 0
+          sleep 1
+        end
+        # Wait this many, then if the queue is still empty, go ahead
+        Minion.message_count(queue_name) == 0
+      end
+    end
+  end
+end

data/lib/mb-minion/message.rb

@@ -0,0 +1,58 @@
+require File.join 'active_support', 'core_ext', 'module', 'delegation'
+
+module Minion
+  class Message
+    attr_accessor :content, :callbacks, :headers, :batch
+    delegate :clear, :map, :each, :size, :count, :[], :each_with_index, :cycle, :shuffle, :to => :content
+    
+    def initialize json="{}", header=nil
+      data = decode(json)
+      @headers   = [header]
+      @callbacks = data['callbacks']
+      @content   = data['content']
+      @batch     = data['batch'] || []
+    end
+    
+    def << data
+      @content << data
+    end
+    
+    # Enqueue a job for the next callback in the chain
+    #
+    # @return void
+    def callback
+      headers.clear
+      if callbacks and not callbacks.empty?
+        queue_name = callbacks.shift
+        Minion.enqueue(queue_name, as_json)
+      end
+    end
+
+    private
+    
+    # Decode the json string into a hash.
+    #
+    # @example Decode the json.
+    #   decode("{ field : "value" }")
+    #
+    # @param [ String ] json The json string.
+    #
+    # @return [ Hash ] The json as a hash.
+    def decode(json)
+      defined?(ActiveSupport::JSON) ?
+        ActiveSupport::JSON.decode(json) : JSON.load(json)
+    end
+    
+    def as_json
+      { 'callbacks' => callbacks,
+        'headers' => headers,
+        'content' => content
+      }
+    end
+    
+    def to_json
+      JSON.dump(as_json || {}).force_encoding("ISO-8859-1")
+    end
+    
+  end
+end

data/lib/mb-minion/version.rb

@@ -0,0 +1,4 @@
+# encoding: utf-8
+module Minion #:nodoc
+  VERSION = "0.2.0"
+end

data/spec/minion/handler_spec.rb

@@ -0,0 +1,147 @@
+require "spec_helper"
+
+describe Minion::Handler do
+
+  before(:all) do
+    Minion.logger {}
+  end
+
+  let(:channel) do
+    stub.quacks_like(AMQP::Channel.allocate)
+  end
+
+  let(:queue) do
+    stub.quacks_like(AMQP::Queue.allocate)
+  end
+
+  describe "#execute" do
+
+    before do
+      channel.expects(:prefetch).at_most_once
+      AMQP::Channel.stubs(:new).returns(channel)
+    end
+
+    context "when the queue is subscribable" do
+
+      let(:handler) do
+        described_class.new("minion.test", lambda{ true })
+      end
+
+      context "when the handler is not already running" do
+
+        before do
+          channel.expects(:queue).with(
+            "minion.test", :durable => true, :auto_delete => false
+          ).returns(queue)
+        end
+
+        it "subscribes to the queue" do
+          queue.expects(:subscribe)
+          handler.execute
+        end
+      end
+
+      context "when the handler is running" do
+
+        before do
+          handler.instance_variable_set(:@running, true)
+        end
+
+        it "does not subscribe again" do
+          channel.expects(:queue).never
+          handler.execute
+        end
+      end
+    end
+
+    context "when the queue is not subscribable" do
+
+      let(:handler) do
+        described_class.new("minion.test", lambda{ true }, :when => lambda{ false })
+      end
+
+      before do
+        channel.expects(:queue).with(
+          "minion.test", :durable => true, :auto_delete => false
+        ).returns(queue)
+      end
+
+      it "unsubscribes from the queue" do
+        queue.expects(:unsubscribe)
+        handler.execute
+      end
+    end
+    
+    context "when wait parameter is specified" do
+      it "should raise an error" do
+        expect do
+          described_class.new("minion.test", lambda{ |batch| {"content" => true} }, :wait => true)
+        end.to raise_error(ArgumentError)
+      end
+    end
+    
+    context "when a batch size is specified" do
+      let(:block) do
+        lambda{ |batch| {"content" => true} }
+      end
+      
+      let(:handler) do
+        described_class.new("minion.test", block, :batch_size => 10)
+      end
+      
+      let(:header) do
+        stub.quacks_like(AMQP::Header.allocate)
+      end
+      
+      let(:serialized) do
+        '{"content":{"field":"value"}}'
+      end
+      
+      let(:batch) do
+        [header, serialized] * 10
+      end
+            
+      before do
+        queue.expects(:subscribe).multiple_yields(batch)
+        channel.expects(:queue).with(
+          "minion.test", :durable => true, :auto_delete => false
+        ).returns(queue)
+        Minion.expects(:execute_handlers)
+        header.expects(:ack)
+      end
+      
+      it "calls once for 10 messages" do
+        block.expects(:call).once
+        handler.execute
+      end
+      
+      context "when wait parameter is specified" do
+        let(:handler) do
+          described_class.new("minion.test", block, :batch_size => 10, :wait => true)
+        end
+        
+        it "doesn't call for 9 messages" do
+          block.expects(:call).never
+          handler.execute
+        end
+      end
+      
+      context "when wait paramter is numeric" do
+        let(:batch) do
+          [header, serialized] * 9
+        end
+        
+        let(:handler) do
+          described_class.new("minion.test", block, :batch_size => 10, :wait => 2)
+        end
+        
+        it "calls once for 9 messages after waiting a bit" do          
+          block.expects(:call).once
+          handler.execute
+        end
+      end
+    end
+    
+    
+  end
+end

data/spec/minion/message_spec.rb

@@ -0,0 +1,38 @@
+require "spec_helper"
+
+describe Minion::Message do
+  let(:header) do
+    stub.quacks_like(AMQP::Header.allocate)
+  end
+  
+  let(:serialized) do
+    '{"content":{"field":"value"}, "callbacks":["minion.second", "minion.third"]}'
+  end
+  
+  subject do
+    Minion::Message.new(serialized, header)
+  end
+  
+  its(:content){ should eql({"field"=>"value"}) }
+  its(:callbacks){ should eql ["minion.second", "minion.third"] }
+  its(:headers){ should eql [header]}
+  
+  context "when callback is executed" do
+    
+    let(:data) do
+      {'callbacks' => ['minion.third'], 'headers' => [], 'content' => {'field' => 'value'}}
+    end
+  
+    before do
+      subject.headers.clear
+      subject.headers.expects(:clear)
+    end
+    
+    it "should enqueue the next job" do
+      Minion.expects(:enqueue).with('minion.second', data)
+      subject.callback
+    end
+    
+  end
+  
+end

data/spec/minion_spec.rb

@@ -0,0 +1,238 @@
+# encoding: utf-8
+require "spec_helper"
+
+describe Minion do
+
+  let(:bunny) do
+    Bunny.new(Minion.config).tap do |bunny|
+      bunny.start
+    end
+  end
+
+  before(:all) do
+    Minion.logger {}
+  end
+
+  describe ".alert" do
+
+    context "when an error handler is provided" do
+
+      let(:error) do
+        RuntimeError.new("testing")
+      end
+
+      before do
+        Minion.error do |error|
+          error.message
+        end
+      end
+
+      after do
+        Minion.error
+      end
+
+      it "delegates to the handler" do
+        Minion.alert(error).should == "testing"
+      end
+    end
+
+    context "when an error handler is not provided" do
+
+      let(:error) do
+        RuntimeError.new("testing")
+      end
+
+      it "raises the error" do
+        expect { Minion.alert(error) }.to raise_error(RuntimeError)
+      end
+    end
+  end
+
+  describe ".enqueue" do
+
+    let(:queue) do
+      bunny.queue("minion.test", :durable => true, :auto_delete => false)
+    end
+
+    before do
+      queue.purge
+    end
+
+    context "when provided a string" do
+
+      context "when no data is provided" do
+
+        before do
+          Minion.enqueue("minion.test")
+        end
+
+        let(:message) do
+          JSON.parse(queue.pop[:payload])
+        end
+
+        it "adds empty json to the queue" do
+          message.should == {"content" => {}}
+        end
+      end
+
+      context "when nil data is provided" do
+
+        before do
+          Minion.enqueue("minion.test", nil)
+        end
+
+        let(:message) do
+          JSON.parse(queue.pop[:payload])
+        end
+
+        it "adds empty json to the queue" do
+          message.should == {"content" => nil}
+        end
+      end
+
+      context "when data is provided" do
+
+        context "when the data has no special characters" do
+
+          let(:data) do
+            {"content"=>{"field"=>"value"}}
+          end
+
+          before do
+            Minion.enqueue("minion.test", data)
+          end
+
+          let(:message) do
+            JSON.parse(queue.pop[:payload])
+          end
+
+          it "adds the json to the queue" do
+            message.should == data
+          end
+        end
+
+        context "when the data contains special characters" do
+
+          let(:data) do
+            {"content"=>{"field"=>"öüäßÖÜÄ"}}
+          end
+
+          before do
+            Minion.enqueue("minion.test", data)
+          end
+
+          let(:message) do
+            JSON.parse(queue.pop[:payload])
+          end
+
+          it "adds the json to the queue" do
+            message.should == data
+          end
+        end
+      end
+    end
+
+    context "when provided a nil queue" do
+
+      it "raises an error" do
+        expect { Minion.enqueue(nil, {}) }.to raise_error(RuntimeError)
+      end
+    end
+
+    context "when passed an array" do
+
+      let(:first) do
+        bunny.queue("minion.first", :durable => true, :auto_delete => false)
+      end
+
+      before do
+        first.purge
+      end
+
+      context "when the array is empty" do
+
+        it "raises an error" do
+          expect { Minion.enqueue([], {}) }.to raise_error(RuntimeError)
+        end
+      end
+
+      context "when the array has queue names" do
+
+        let(:data) do
+          {"field"=>"value"}
+        end
+        
+        let(:serialized) do
+          {"content"=>{"field"=>"value"}, "callbacks"=>["minion.second", "minion.third"]}
+        end
+
+        before do
+          Minion.enqueue([ "minion.first", "minion.second", "minion.third" ], data)
+        end
+
+        let(:message) do
+          JSON.parse(first.pop[:payload])
+        end
+
+        it "adds the serialized data to the first queue" do
+          message.should == serialized
+        end
+        
+        context "when the data has already been serialized" do
+          before do
+            Minion.enqueue([ "minion.first", "minion.second", "minion.third" ], serialized)
+          end
+          
+          let(:message) do
+            JSON.parse(first.pop[:payload])
+          end
+          
+          it "adds has the same serialized data in the first queue" do
+            message.should == serialized
+          end
+        end
+      end
+    end
+  end
+
+
+  describe ".error_handling" do
+
+    context "when nothing has been defined" do
+
+      it "returns nil" do
+        Minion.send(:error_handling).should be_nil
+      end
+    end
+  end
+
+  describe ".error" do
+
+    let(:block) do
+      lambda{ "testing" }
+    end
+
+    before do
+      Minion.error(&block)
+    end
+
+    it "sets the error handling to the provided block" do
+      Minion.send(:error_handling).should == block
+    end
+  end
+
+  describe ".info" do
+
+    let(:block) do
+      lambda{ |message| message }
+    end
+
+    before do
+      Minion.logger(&block)
+    end
+
+    it "delegates the logging to the provided block" do
+      Minion.info("testing").should == "testing"
+    end
+  end
+end