mbus 1.0.0

data/lib/messagebus/swarm/drone.rb

@@ -0,0 +1,195 @@
+# Copyright (c) 2012, Groupon, Inc.
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+# Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+# 
+# Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# 
+# Neither the name of GROUPON nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+require 'json'
+
+module Messagebus
+  module Swarm
+    ##
+    # This is a composition of a consumer and a separate message processor.
+    # It allows you to use Plain-Old-Ruby-Objects to do message processing.
+    # See #initialize for how the messages are delegated.
+    class Drone
+      # Raise this error when you want to abort processing a message and not
+      # acknowledge it.
+      class AbortProcessing < StandardError; end
+
+      INITIALIZING = "initializing"
+      RECEIVING    = "receiving"
+      PROCESSING   = "processing"
+      COMPLETED    = "completed"
+
+      STOP_PROCESSING_MESSAGE = "stop processing message"
+
+      attr_reader :state, :id
+
+      ##
+      # Expected options:
+      # * :ack_on_error (default false): whether to ack the message when an error was raised. Aliased to auto_acknowledge for backwards compatibility
+      # * :consumer (required): a consumer object for that topic
+      # * :destination_name (required): the message bus queue/topic name
+      # * :id: an id for this drone (just used for debugging)
+      # * :logger (required): the logger to publish messages to
+      # * :worker_class (required): the actual worker that will be used to do the processing
+      #
+      # As messages come down, they will be passed to the worker's perform or
+      # perform_on_destination method. The methods will be called in the
+      # following priority (if they exist):
+      # * perform_on_destination(message_payload, destination_message_came_from)
+      # * perform(message_payload)
+      #
+      # A message is processed by:
+      #   message = receive_message
+      #   begin
+      #     worker.perform(message)
+      #     ack(message)
+      #   rescue AbortProcessing
+      #     # raise this error if you want to say "Don't ack me"
+      #   rescue StandardError
+      #     ack(message) if ack_on_error
+      #   end
+      def initialize(options)
+        @auto_acknowledge = options.fetch(:ack_on_error, options[:auto_acknowledge])
+        @consumer, @destination_name, @worker_class, @id, @logger = options.values_at(:consumer, :destination_name, :worker_class, :id, :logger)
+        @state = INITIALIZING
+        @logger.debug { "initializing a drone drone_id=#{@id}" }
+      end
+
+      ##
+      # This is the main loop for the drone's work. This will not return until
+      # the drone is stopped via #stop.
+      #
+      # If the consumer hasn't been started yet, this method will start it. It
+      # also will auto close the consumer in that case.
+      def processing_loop
+        @processing = true
+
+        auto_started_consumer = false
+        begin
+          if !@consumer.started?
+            @logger.debug "auto starting the consumer drone_id=#{@id}"
+            @consumer.start
+            auto_started_consumer = true
+          end
+
+          while @processing
+            message = nil
+            begin
+              @logger.debug "waiting for a message"
+
+              @state = RECEIVING
+              message = @consumer.receive
+              # intentional === here, this is used as a signal, so we can use object equality
+              # to check if we received the signal
+              if message === STOP_PROCESSING_MESSAGE
+                @logger.info "received a stop message, exiting drone_id=#{@id}, message=#{message.inspect}"
+                @state = COMPLETED
+                next
+              end
+
+              @logger.info "received message drone_id=#{@id}, message_id=#{message.message_id}"
+
+              @state = PROCESSING
+              @logger.info "processing message drone_id=#{@id}, message_id=#{message.message_id}, worker=#{@worker_class.name}"
+
+              raw_message = extract_underlying_message_body(message)
+              @logger.debug { "drone_id=#{@id} message_id=#{message.message_id}, message=#{raw_message.inspect}" }
+
+              worker_perform(@logger, @destination_name, @worker_class, @consumer, @auto_acknowledge, message, raw_message)
+
+              @state = COMPLETED
+            rescue => except
+              @logger.warn "exception processing message drone_id=#{@id}, message_id=#{message && message.message_id}, exception=\"#{except.message}\", exception_backtrace=#{except.backtrace.join("|")}"
+            end
+          end
+        ensure
+          if auto_started_consumer
+            @logger.debug "auto stopping the consumer drone_id=#{@id}"
+            @consumer.stop
+          end
+        end
+
+        @logger.info("gracefully exited drone_id=#{@id}")
+      end
+
+      ##
+      # Stops this drone from processing any additional jobs.
+      # This will not abort any in progress jobs.
+      def stop
+        @logger.info("received stop message, current_state=#{@state}, processing=#{@processing}, drone_id=#{@id}")
+        return if !@processing
+
+        @processing = false
+        @consumer.insert_sentinel_value(STOP_PROCESSING_MESSAGE)
+      end
+
+      private
+      def blocking_on_receive?(state)
+        state == RECEIVING
+      end
+
+      def worker_perform(logger, destination_name, worker_class, consumer, auto_acknowledge, message, raw_message)
+        begin
+          if worker_class.respond_to?("perform_on_destination")
+            worker_class.perform_on_destination(raw_message, destination_name)
+          else
+            worker_class.perform(raw_message)
+          end
+
+          # acknowledge unless exception is thrown (auto perform == true)
+          logger.debug "processing complete, acknowledging message, drone_id=#{@id}, message_id=#{message.message_id}"
+          consumer.ack
+
+        # This is just a clean error people can throw to trigger an abort
+        rescue AbortProcessing => e
+          logger.info "processing aborted drone_id=#{@id}, message_id=#{message.message_id}"
+        rescue => worker_exception
+          logger.warn "processing failed drone_id=#{@id}, message_id=#{message.message_id} exception=\"#{worker_exception.message}\", exception_backtrace=#{worker_exception.backtrace.join("|")}"
+
+          if auto_acknowledge
+            logger.info "despite failure, auto acknowledging message, drone_id=#{@id}, message_id=#{message.message_id}"
+            consumer.ack
+          end
+        end
+      end
+
+      def extract_underlying_message_body(message)
+        payload = message.raw_message.payload
+        if payload.json?
+          JSON.parse(payload.stringPayload)
+        elsif payload.binary?
+          payload.binaryPayload
+        elsif payload.string?
+          payload.stringPayload
+        end
+      end
+    end
+  end
+end

data/lib/messagebus/swarm/drone/logging_worker.rb

@@ -0,0 +1,53 @@
+# Copyright (c) 2012, Groupon, Inc.
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+# Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+# 
+# Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# 
+# Neither the name of GROUPON nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+require 'json'
+
+module Messagebus
+  module Swarm
+    class Drone
+      ##
+      # Use this for easy testing that a messagebus message is received.
+      # Example config:
+      #  -
+      #   :destination: jms.topic.some_destination_you_want_to_debug
+      #   :subscription_id: <some_subscription_id>
+      #   :worker: Messagebus::Swarm::Drone::LoggingWorker
+      #   :drones: 1
+      class LoggingWorker
+        def self.perform_on_destination(message, destination)
+          log_message = %|received a message. destination=#{destination}, message=#{message.inspect}|
+          Rails.logger.info(log_message) if defined?(Rails.logger)
+          puts log_message
+        end
+      end
+    end
+  end
+end

data/lib/messagebus/validations.rb

@@ -0,0 +1,68 @@
+# Copyright (c) 2012, Groupon, Inc.
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+# Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+# 
+# Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# 
+# Neither the name of GROUPON nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+module Messagebus
+  # :nodoc:all
+  module Validations
+    def valid_host?(string)
+      Messagebus::SERVER_REGEX.match(string)
+    end
+
+    def validate_destination_config(name, is_consumer = false, options = {})
+      raise InvalidDestination.new("destination name is nil") unless name
+
+      if is_consumer && name.match(/^jms.topic/) && options[:subscription_id].nil?
+        raise InvalidDestination.new("destination type TOPIC requires a subscription_id")
+      end
+    end
+
+    def validate_connection_config(host_params, options = {})
+      if options[:ack_type] &&
+        options[:ack_type] != Messagebus::ACK_TYPE_AUTO_CLIENT &&
+        options[:ack_type] != Messagebus::ACK_TYPE_CLIENT
+        raise InvalidAcknowledgementType.new(options[:ack_type])
+      end
+
+      if host_params.nil?
+        raise InvalidHost.new(host_params)
+      end
+
+      if host_params.is_a?(Array)
+        host_params.each do |string|
+          unless valid_host?(string)
+            raise InvalidHost.new("host should be defined as <host>:<port>, received: #{host_params}")
+          end
+        end
+      else
+        raise InvalidHost.new("host should be defined as <host>:<port>, received #{host_params}")
+      end
+    end
+  end
+end

data/lib/messagebus/version.rb

@@ -0,0 +1,36 @@
+# Copyright (c) 2012, Groupon, Inc.
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+# Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+# 
+# Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# 
+# Neither the name of GROUPON nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+module Messagebus
+  MAJOR = 1
+  MINOR = 0
+  PATCH = 0
+  VERSION = "#{MAJOR}.#{MINOR}.#{PATCH}"
+end

data/messagebus.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+require "#{File.dirname(__FILE__)}/lib/messagebus/version"
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Pradeep Jawahar", "Lin Zhao", "Erik Weathers"]
+  gem.email         = ["pjawahar@groupon.com", "lin@groupon.com", "eweathers@groupon.com"]
+  gem.description   = %q{Messagebus integration gem}
+  gem.summary       = %q{Messagebus Client}
+  gem.homepage      = "https://github.com/groupon/Message-Bus"
+
+  gem.executables   = %w(messagebus_swarm)
+  gem.files         = Dir['bin/*'] + Dir["lib/**/*.rb"] + Dir["vendor/**/*"] + %w(README.mediawiki Rakefile messagebus.gemspec)
+  gem.test_files    = Dir["spec*/**/*.rb"]
+  gem.name          = "mbus"
+  gem.require_paths = ["vendor/gems", "lib"]
+  gem.version       = Messagebus::VERSION
+  gem.license       = 'BSD'
+
+  gem.required_rubygems_version = ">= 1.3.6"
+
+  if RUBY_VERSION < "1.9"
+    # json is built into ruby 1.9
+    gem.add_dependency "json"
+  end
+  gem.add_dependency "thrift", "0.9.0"
+
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "simplecov"
+end

data/spec/messagebus/client_spec.rb

@@ -0,0 +1,157 @@
+require File.join(File.dirname(__FILE__), '..', 'spec_helper')
+
+describe Messagebus::Client do
+  before do
+    @config = {
+      "enable_auto_init_connections" => true,
+      "clusters" => [
+        {
+          "type" => "producer-cluster",
+          "address" => "localhost:61613",
+          "destinations" => [
+            "jms.queue.testQueue1",
+            "jms.topic.testTopic1"
+          ]
+        }
+      ]
+    }
+
+    @logger = mock(Logger, :info => true, :error => true, :debug => true, :warn => true)
+    Logger.stub(:new).and_return(@logger)
+
+    @cluster_map = mock(Messagebus::ClusterMap, :start => true, :stop => true)
+    Messagebus::ClusterMap.stub!(:new).and_return(@cluster_map)
+    
+  end
+
+  describe "constructing a client" do
+    describe "when creating a singleton logger" do
+      it "creates a logger using tmp directory if configuration is omitted" do
+        Messagebus::Client.new(@config)
+        Messagebus::Client.logger.should == @logger
+      end
+
+      it "creates a logger with the supplied log_file" do
+        Logger.should_receive(:new).with("log/messagebus-client.log").and_return(@logger)
+        Messagebus::Client.new(@config.merge("log_file" => "log/messagebus-client.log"))
+      end
+    end
+
+    it "automatically starts the cluster if configured" do
+      @cluster_map.should_receive(:start)
+      Messagebus::Client.new(@config.merge("enable_auto_init_connections" => true)).start
+    end
+
+    it "automatically defaults to cluster not started" do
+      @cluster_map.should_not_receive(:start)
+      Messagebus::Client.new(@config)
+    end
+  end
+
+  describe ".start (class method)" do
+    it "auto closes the connection when a block is given" do
+      @cluster_map.should_receive(:start)
+      @cluster_map.should_receive(:stop)
+      Messagebus::Client.start(@config.merge("enable_auto_init_connections" => true)) {}
+    end
+
+    it "makes sure it stops if the block errors out" do
+      @cluster_map.should_receive(:start)
+      @cluster_map.should_receive(:stop)
+      proc do
+        Messagebus::Client.start(@config.merge("enable_auto_init_connections" => true)) do
+          raise "error123"
+        end
+      end.should raise_error("error123")
+    end
+
+    it "doesn't auto close if no block passed" do
+      @cluster_map.should_receive(:start)
+      @cluster_map.should_not_receive(:stop)
+      Messagebus::Client.start(@config.merge("enable_auto_init_connections" => true))
+
+    end
+  end
+
+  describe "#start" do
+    it "delegates to the cluster map start" do
+      @cluster_map.should_receive(:start)
+      @cluster_map.should_not_receive(:stop)
+      Messagebus::Client.new(@config.merge("enable_auto_init_connections" => true)).start
+    end
+  end
+
+  describe "#stop" do
+    it "delegates to the cluster map stop" do
+      @cluster_map.should_receive(:stop)
+      Messagebus::Client.new(@config).stop
+    end
+  end
+
+  describe "#logger" do
+    it "shares the logger across instances" do
+      Messagebus::Client.new(@config).logger.should be(Messagebus::Client.new(@config).logger)
+    end
+  end
+
+  describe "#publish" do
+    before do
+      @producer  = mock(Messagebus::Producer, :publish => true)
+      @cluster_map.stub!(:find).and_return(@producer)
+      @producer.stub!(:started?).and_return(true)
+
+      @message = mock(Messagebus::Message, :create => true, :message_id => "ee4ae017f409dc3f4aad38dddeb7bf88")
+      Messagebus::Message.stub!(:create).and_return(@message)
+    end
+
+    it "creates a message from the provided object" do
+      Messagebus::Message.should_receive(:create).with({:benjamin => :franklins})
+      Messagebus::Client.new(@config).publish("jms.queue.Testqueue1", {:benjamin => :franklins})
+    end
+
+    it "requires an existing destination" do
+      @producer.should_not_receive(:publish)
+      lambda {
+        @cluster_map.stub!(:find).and_return(nil)
+        Messagebus::Client.new(@config).publish("jms.queue.testQueue1", {:benjamin => :franklins})
+      }.should raise_error(Messagebus::Client::InvalidDestinationError)
+    end
+
+    it "sends a safe message by default" do
+      @producer.should_receive(:publish)
+      Messagebus::Client.new(@config).publish("jms.queue.testQueue1", {:benjamin => :franklins})
+    end
+
+    it "sends an unsafe message by when the safe parameter is false" do
+      @producer.should_receive(:publish)
+      Messagebus::Client.new(@config).publish("jms.queue.testQueue1", {:benjamin => :franklins}, 0, false)
+    end
+    
+    it "sends a message with headers" do
+      headers = {"priority" => 6}
+      @producer.should_receive(:publish).with("jms.queue.testQueue1", @message, headers, false)
+      Messagebus::Client.new(@config).publish("jms.queue.testQueue1", {:benjamin => :franklins}, 0, false, false, headers)
+    end
+  end
+  
+  describe "#reload_config_on_interval" do 
+   it "reloads config when the interval is reached" do
+     @cluster_map.should_receive(:update_config).with(@config)
+     Messagebus::Client.new(@config).reload_config_on_interval(@config,0)
+   end
+  end
+
+  describe "#headers" do
+    it "creates a header from current time and the delay milliseconds provided to publish" do
+      delay = 3
+      time = Time.now
+      Time.stub!(:now).and_return(time)
+
+      client = Messagebus::Client.new(@config)
+      client.headers(delay).should == {
+        Messagebus::Producer::SCHEDULED_DELIVERY_TIME_MS_HEADER => ((time.to_i * 1000) + delay).to_s
+      }
+    end
+  end
+end
+