lolitra 0.2.0 → 0.2.1

data/README.md

@@ -4,7 +4,7 @@ Amqp and Faye event bus
 
 ## Installation
 
-Add this line to your application's Gemfile:
+Add this line to your Gemfile:
 
     gem 'lolitra'
 
@@ -20,14 +20,13 @@ Or install it yourself as:
 
 Create an event bus on initializers
 
-    Lolitra::AmqpBus.new(
+    Lolitra::RabbitmqBus.new(
       :exchange => "exchangetest",
-      :queue_prefix => "my_prefix_",
       :host => "127.0.0.1",
       :port => 5672,
       :user => "guest",
       :pass => "guest",
-      :pull_subscribers => [DevicesHandler, FoldersHandler, UsersHandler]
+      :pull_subscribers => [DevicesHandler]
     )
 
 Create messages
@@ -68,6 +67,34 @@ Create a message handler
        
     end
 
+*Rabbitmq*
+Lolitra generates a deadletter exchange and queues to handle dead letters and will be aware about connections issues, reconnecting on every failure.
+
+*Deadletter manual handling*
+With lolitra you can recover dead letter message with irb or rails console.
+
+```
+Lolitra::subscribers 
+```
+will return the available handlers
+
+```
+Lolitra::process_deadletters(DevicesHandler)
+```
+will process all dead letter from DevicesHandler until found an exception
+Fail recover deadletter will reenqueue to dead letter queue
+
+```
+Lolitra::remove_next_deadletter(DeviceHandler)
+```
+Will remove next deadletter without doing anything
+
+```
+Lolitra::purge_deadletters(DeviceHandler)
+```
+Will remove all deadletters without doing anything
+
+
 ## Contributing
 
 1. Fork it

data/lib/lolitra/handler_base.rb

@@ -3,6 +3,8 @@ require 'log4r'
 require 'amqp'
 require 'amqp/utilities/event_loop_helper'
 require 'json'
+require 'fileutils'
+require_relative 'rabbitmq_bus'
 
 module Lolitra
   include Log4r
@@ -27,6 +29,32 @@ module Lolitra
     Lolitra::MessageHandlerManager.publish(message)
   end
 
+  def self.unsubscribe(handler_class, &block)
+    Lolitra::MessageHandlerManager.unsubscribe(handler_class, &block)
+  end
+
+  def self.disconnect(&block)
+    Lolitra::MessageHandlerManager.disconnect(&block)
+  end
+
+  def self.subscribers
+    Lolitra::MessageHandlerManager.instance.subscribers.collect do |subscriber|
+      subscriber.name
+    end
+  end
+
+  def self.process_deadletters(subscriber)
+    Lolitra::MessageHandlerManager.instance.process_deadletters(subscriber)
+  end
+
+  def self.remove_next_deadletter(subscriber)
+    Lolitra::MessageHandlerManager.instance.remove_next_deadletter(subscriber)
+  end
+
+  def self.purge_deadletters(subscriber)
+    Lolitra::MessageHandlerManager.instance.purge_deadletters(subscriber)
+  end
+
   module MessageHandler
     module Helpers
       def self.underscore(arg)
@@ -65,11 +93,7 @@ module Lolitra
       end
 
       def handle(message)
-        begin      
-          get_handler(message).handle(message)
-        rescue => e
-          Lolitra::log_exception(e)
-        end
+        get_handler(message).handle(message)
       end
       
       def publish(message)
@@ -142,6 +166,7 @@ module Lolitra
     include Singleton
 
     attr_accessor :bus
+    attr_accessor :subscribers
 
     def self.bus=(new_bus)
       instance.bus = new_bus
@@ -155,20 +180,36 @@ module Lolitra
       instance.register_subscriber(handler_class)
     end
    
+    def subscribers
+      @subscribers ||= []
+    end
+
+    def process_deadletters(handler_class)
+      bus.process_deadletters(handler_class)
+    end
+
+    def purge_deadletters(handler_class)
+      bus.purge_deadletters(handler_class)
+    end
+
+    def remove_next_deadletter(handler_class)
+      bus.remove_next_deadletter(handler_class)
+    end
+
     def register_subscriber(handler_class)
+      subscribers << handler_class
       handler_class.handle_messages.each do |message_class|
         bus.subscribe(message_class, handler_class)
       end
     end
 
-    def self.register_pull_subscriber(handler_class)
-      instance.register_pull_subscriber(handler_class)
+    def self.unsubscribe(handler_class, &block)
+      instance.unsubscribe(handler_class, &block)
     end
-   
-    def register_pull_subscriber(handler_class)
-      handler_class.handle_messages.each do |message_class|
-        bus.pull_subscribe(message_class, handler_class)
-      end
+
+    def unsubscribe(handler_class, &block)
+      Lolitra::logger.info("Unsubscribing #{handler_class}")
+      bus.unsubscribe(handler_class, &block)
     end
 
     def self.publish(message)
@@ -180,6 +221,14 @@ module Lolitra
       Lolitra::logger.debug("#{message.marshall}")
       bus.publish(message)
     end
+
+    def self.disconnect(&block)
+      instance.disconnect(&block)
+    end
+
+    def disconnect(&block)
+      bus.disconnect(&block)
+    end
   end
 
   module Message
@@ -241,89 +290,4 @@ module Lolitra
       @socketClient.publish(message.class.message_key, message.marshall)
     end 
   end
-
-  class AmqpBus
-    attr_accessor :queue_prefix
-    attr_accessor :connection
-    attr_accessor :exchange
-
-    def initialize(hash = {})
-      Lolitra::MessageHandlerManager.bus = self
-
-      @channels = {}
-      @params = hash.reject { |key, value| !value }
-      raise "no :exchange specified" unless hash[:exchange]
-
-      self.queue_prefix = hash[:queue_prefix]||""
-      AMQP::Utilities::EventLoopHelper.run do
-        self.connection = AMQP.start(@params) do |connection|
-          channel = create_channel(connection) do |channel|
-            begin
-              self.exchange = channel.topic(@params[:exchange], :durable => true)
-
-              @params[:pull_subscribers].each do |handler|
-                Lolitra::MessageHandlerManager.register_pull_subscriber(handler)
-              end
-            rescue => e
-              Lolitra::logger.debug("error")
-              Lolitra::log_exception(e)
-            end
-          end
-        end
-      end
-    end
-
-    def subscribe(message_class, handler_class)
-      create_queue(message_class, handler_class, {:exclusive => true, :durable => false}, "")
-    end
-
-    def pull_subscribe(message_class, handler_class)
-      create_queue(message_class, handler_class, {:durable => true})
-    end
-
-    def publish(message)
-      #TODO: if exchange channel is closed doesn't log anything
-      self.exchange.publish(message.marshall, :routing_key => message.class.message_key, :timestamp => Time.now.to_i)
-    end
-
-  private
-    def create_channel(connection, &block)
-      channel = AMQP::Channel.new(connection) do
-        channel.on_error do |channel, close|
-          Lolitra::logger.error("Channel error: #{channel}")
-          Lolitra::logger.error(close)
-        end
-        block.call(channel)
-      end
-      channel 
-    end
-
-    def create_queue(message_class, handler_class, options)
-      begin
-        queue_name = queue_prefix + MessageHandler::Helpers.underscore(handler_class.name)
-
-        create_channel(self.connection) do |channel|
-          channel.queue(queue_name, options).bind(self.exchange, :routing_key => message_class.message_key)
-          channel.close
-        end
-      
-        if !@channels[queue_name] #Only one subscriber by queue_name
-          @channels[queue_name] = create_channel(self.connection) do |channel|
-            channel.prefetch(1).queue(queue_name, options).subscribe do |info, payload|
-              begin
-                Lolitra::logger.debug("Message recived: #{info.routing_key}")
-                Lolitra::logger.debug("#{payload}")
-                message_class_tmp = handler_class.handlers[info.routing_key][0]
-                handler_class.handle(message_class_tmp.unmarshall(payload))
-              rescue => e
-                Lolitra::log_exception(e)
-              end
-            end
-          end
-        end
-      rescue => e
-          Lolitra::log_exception(e)
-      end
-    end
-  end
 end  

data/lib/lolitra/rabbitmq_bus.rb

@@ -0,0 +1,235 @@
+module Lolitra
+  class RabbitmqBus
+    attr_accessor :connection
+    attr_accessor :exchange
+    attr_accessor :exchange_dead_letter
+    attr_accessor :options
+    attr_accessor :subscribers
+
+    SUBSCRIBE_OPTIONS = {:durable => true}
+
+    def initialize(hash = {})
+      Lolitra::MessageHandlerManager.bus = self
+
+      self.options = {
+        :queue_prefix => "",
+        :queue_suffix => "",
+        :exchange_dead_suffix => ".dead",
+        :exchange_dead_params => {}, 
+        :queue_params => {},
+        :queue_dead_suffix => ".dead",
+        :queue_dead_params => {},
+        :no_consume => false,
+      }.merge(hash.delete(:options) || {})
+
+      self.options[:queue_params][:arguments] = {} unless self.options[:queue_params][:arguments]
+
+      self.options[:queue_params][:arguments] = {
+        "x-dead-letter-exchange" => "#{hash[:exchange]}#{@options[:exchange_dead_suffix]}"
+      }.merge(self.options[:queue_params][:arguments])
+
+      @channels = {}
+      @params = hash.reject { |key, value| !value }
+      raise "no :exchange specified" unless hash[:exchange]
+
+      AMQP::Utilities::EventLoopHelper.run do
+        self.connection = AMQP.start(@params) do |connection|
+          Lolitra::logger.info("Connected to rabbitmq.")
+          channel = create_channel(connection) do |channel|
+            begin
+              self.exchange = channel.topic(@params[:exchange], :durable => true)
+              self.exchange_dead_letter = channel.topic("#{@params[:exchange]}#{@options[:exchange_dead_suffix]}", :durable => true)
+
+              @params[:subscribers].each do |handler|
+                Lolitra::MessageHandlerManager.register_subscriber(handler)
+              end
+            rescue => e
+              Lolitra::log_exception(e)
+            end
+          end
+        end
+        self.connection.on_tcp_connection_loss do |connection, settings|
+          # reconnect in 10 seconds, without enforcement
+          Lolitra::logger.info("Connection loss. Trying to reconnect in 10 secs if needed.")
+          connection.reconnect(false, 10)
+        end
+      end
+    end
+
+    def disconnect(&block)
+      self.connection.close(&block)
+    end
+
+    def subscribe(message_class, handler_class)
+      create_queue(message_class, handler_class, SUBSCRIBE_OPTIONS)
+    end
+
+    def publish(message)
+      #TODO: if exchange channel is closed doesn't log anything
+      self.exchange.publish(message.marshall, :routing_key => message.class.message_key, :timestamp => Time.now.to_i)
+    end
+
+    def unsubscribe(handler_class, &block)
+      queue_name = generate_queue_name(handler_class)
+      begin
+        create_channel(self.connection) do |channel|
+          queue = channel.queue(queue_name, SUBSCRIBE_OPTIONS) do |queue|
+            begin
+              queue.delete
+              block.call(handler_class, true)
+            rescue => e
+              Lolitra::log_exception(e)
+              block.call(handler_class, false)
+            end
+          end
+        end
+      rescue => e
+        Lolitra::log_exception(e)
+      end
+    end
+
+    def process_deadletters(handler_class)
+      queue_name_dead = generate_queue_name_dead(handler_class)
+      options = SUBSCRIBE_OPTIONS 
+      create_channel(self.connection) do |channel|
+        begin
+          channel.queue(queue_name_dead, options.merge(@options[:queue_dead_params])) do |queue|
+            recursive_pop(channel, queue, handler_class)
+          end
+        rescue => e
+          Lolitra::log_exception(e)
+        end
+      end
+      true
+    end
+
+    def purge_deadletters(handler_class)
+      queue_name_dead = generate_queue_name_dead(handler_class)
+      options = SUBSCRIBE_OPTIONS 
+      create_channel(self.connection) do |channel|
+        begin
+          channel.queue(queue_name_dead, options.merge(@options[:queue_dead_params])) do |queue|
+            purge_queue(queue)
+          end
+        rescue => e
+          Lolitra::log_exception(e)
+        end
+      end
+      true
+    end
+
+    def remove_next_deadletter(handler_class)
+      queue_name_dead = generate_queue_name_dead(handler_class)
+      options = SUBSCRIBE_OPTIONS 
+      create_channel(self.connection) do |channel|
+        begin
+          channel.queue(queue_name_dead, options.merge(@options[:queue_dead_params])) do |queue|
+            queue.pop
+          end
+        rescue => e
+          Lolitra::log_exception(e)
+        end
+      end
+      true
+    end
+
+  private
+    def purge_queue(queue)
+      queue.pop do |info, payload|
+        if (payload)
+          purge_queue(queue)
+        end
+      end
+    end
+
+    def recursive_pop(channel, queue, handler_class)
+      queue.pop(:ack => true) do |info, payload|
+        if payload
+          Lolitra::logger.info("Routing key: #{info.routing_key}")
+          Lolitra::logger.info("Payload: #{payload}")
+          begin
+            message_class_tmp = handler_class.handlers[info.routing_key][0]
+            handler_class.handle(message_class_tmp.unmarshall(payload))
+            info.ack
+            recursive_pop(channel, queue, handler_class)
+          rescue => e
+            channel.reject(info.delivery_tag, true)
+            Lolitra::log_exception(e)
+          end
+        end
+      end 
+    end
+
+    def publish_payload(routing_key, payload)
+      self.exchange.publish(payload, :routing_key => routing_key, :timestamp => Time.now.to_i)
+    end
+
+    def create_channel(connection, &block)
+      channel = AMQP::Channel.new(connection, :auto_recovery => true) do
+        channel.on_error do |channel, close|
+          Lolitra::logger.error("Channel error: #{channel}")
+          Lolitra::logger.error(close)
+        end
+        block.call(channel)
+      end
+      channel 
+    end
+
+    def generate_queue_name(handler_class)
+      "#{@options[:queue_prefix]}#{MessageHandler::Helpers.underscore(handler_class.name)}#{@options[:queue_suffix]}"
+    end
+
+    def generate_queue_name_dead(handler_class)
+      "#{generate_queue_name(handler_class)}#{@options[:queue_dead_suffix]}"
+    end
+
+    def create_queue(message_class, handler_class, options)
+      begin
+        queue_name = generate_queue_name(handler_class)
+        queue_name_dead = generate_queue_name_dead(handler_class)
+
+        create_channel(self.connection) do |channel|
+          begin
+            channel.queue(queue_name, options.merge(@options[:queue_params])).bind(self.exchange, :routing_key => message_class.message_key)
+            channel.queue(queue_name_dead, options.merge(@options[:queue_dead_params])).bind(self.exchange_dead_letter, :routing_key => message_class.message_key)
+            channel.close
+          rescue => e
+            Lolitra::log_exception(e)
+          end
+        end
+      
+        if !@options[:no_consume] && !@channels[queue_name] #Only one subscriber by queue_name
+          @channels[queue_name] = subscribe_to_messages(queue_name, options, handler_class)
+        end
+      rescue => e
+          Lolitra::log_exception(e)
+      end
+    end
+
+    def subscribe_to_messages(queue_name, options, handler_class)
+      create_channel(self.connection) do |channel|
+        channel.prefetch(1).queue(queue_name, options.merge(@options[:queue_params])).subscribe(:ack => true) do |info, payload|
+          begin
+            Lolitra::logger.debug("Message recived: #{info.routing_key}")
+            Lolitra::logger.debug("#{payload}")
+            message_class_tmp = handler_class.handlers[info.routing_key][0]
+            handler_class.handle(message_class_tmp.unmarshall(payload))
+            info.ack 
+          rescue => e
+            channel.reject(info.delivery_tag, false)
+            Lolitra::log_exception(e)
+            mark_deadletter
+          end
+        end
+      end
+    end
+
+    def remove_mark_deadletter
+      FileUtils.rm("#{Dir.pwd}/tmp/deadletter")
+    end
+
+    def mark_deadletter
+      FileUtils.touch("#{Dir.pwd}/tmp/deadletter")
+    end
+  end
+end

data/lib/lolitra/version.rb

@@ -1,3 +1,3 @@
 module Lolitra
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 2
-  - 0
-  version: 0.2.0
+  - 1
+  version: 0.2.1
 platform: ruby
 authors: 
 - Hugo Freire
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2015-02-20 00:00:00 +01:00
+date: 2015-02-26 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -102,6 +102,7 @@ files:
 - Rakefile
 - lib/lolitra.rb
 - lib/lolitra/handler_base.rb
+- lib/lolitra/rabbitmq_bus.rb
 - lib/lolitra/version.rb
 - lolitra.gemspec
 - spec/lolitra_spec.rb