liebre 0.1.5 → 0.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b885fad6a4e048ed9330efede57398f9b75e128b
-  data.tar.gz: 082ca325c3b614d6b60bb372a65d2039458d3cfc
+  metadata.gz: 359dbb271508fe21b7d58c5c0a5aa04d98821b98
+  data.tar.gz: 792514c9a04576605d85e66b2cf294b3619eb8de
 SHA512:
-  metadata.gz: 3c967f9aa11caa15105394e29fba511aea2cf53504a5ec8af4324067a7e8fe282f9c7f96a2457a3e6b4b20efbbd8b4e15f6652fce20850bff5c9ac1b365ce974
-  data.tar.gz: d453dc69d000db9c0695d81649189da9e7c4b1a45493d110ee0ebc488960ea9c7e4a6fe089a1ae3d9f68101365ddea79c5fe419ce029cfc4cee34865246a8a63
+  metadata.gz: f182c3482cfe14d0022065449731fd17cdffc20285a59828a0a4928133d84ace37aa1cfe724513519a4016ff226f443d3caf39376a23a53e172d5b157ed05fb6
+  data.tar.gz: bacd99b79ca2d22c1ae19e207bdef34906993f4393cd9a186aec5503b6b1bfd042edfd039226fb023afcf4aaaea2f0a4f51ed990e15e6ef869a0f4d97a3f3041

data/Gemfile.lock

@@ -1,14 +1,14 @@
 PATH
   remote: .
   specs:
-    liebre (0.1.3)
+    liebre (0.1.11)
       bunny (~> 2.5, >= 2.5.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
     amq-protocol (2.0.1)
-    bunny (2.5.1)
+    bunny (2.6.3)
       amq-protocol (>= 2.0.1)
     coderay (1.1.1)
     diff-lcs (1.2.5)
@@ -44,4 +44,4 @@ DEPENDENCIES
   rspec
 
 BUNDLED WITH
-   1.12.5
+   1.13.6

data/lib/liebre/connection_manager.rb

@@ -1,21 +1,33 @@
 require "yaml"
 require "bunny"
+require "singleton"
 
 module Liebre
   class ConnectionManager
 
-    def initialize path = Liebre::Config.connection_path
-      @path = path
+    include Singleton
+
+    def initialize
+      @path = Liebre::Config.connection_path
       @connections = {}
+      @channels = {}
+      @lock = Mutex.new
     end
 
     def start
-      initialize_connections
-      connections.each do |_, bunny|
-        bunny.start
+      @lock.synchronize do
+        initialize_connections
+        connections.each do |connection_name, bunny|
+          begin
+            bunny.start
+          rescue => e
+            $logger.error("#{self.class.name}: Can't connect to #{connection_name} instance")
+            $logger.error(e.message + "\n" + e.backtrace.join("\n"))
+          end
+        end
       end
     end
-    
+
     def ensure_started
       all_open = !@connections.empty? and @connections.all? do |_, bunny|
         bunny.open?
@@ -32,17 +44,27 @@ module Liebre
       connections[connection_name.to_sym]
     end
 
+    def channel_for connection_name, consumer_pool_size = 1
+      @lock.synchronize do
+        channels[connection_name] ||= begin
+          get(connection_name).create_channel nil, consumer_pool_size
+        end
+      end
+    end
+
     def stop
-      connections.each do |_, bunny|
-        if bunny and bunny.open?
-          bunny.close
+      @lock.synchronize do
+        connections.each do |_, bunny|
+          if bunny and bunny.open?
+            bunny.close
+          end
         end
+        connections.clear
       end
-      connections.clear
     end
 
     private
-    
+
     def initialize_connections
       config.each do |name, conf|
         @connections[name.to_sym] = connection_for(conf)
@@ -58,7 +80,7 @@ module Liebre
       Liebre.env ? result.fetch(Liebre.env) : result
     end
 
-    attr_reader :path, :connections
+    attr_reader :path, :connections, :channels
 
   end
 end

data/lib/liebre/publisher.rb

@@ -7,16 +7,17 @@ module Liebre
     
     def enqueue message, options = {}
       with_connection do
-        logger.debug "Liebre: Publishing '#{message}' with '#{options}' to exchange: #{exchange}"
+        exchange = exchange_for default_channel
+        logger.debug "Liebre: Publishing '#{message}' with '#{options}' to exchange: #{exchange.name}"
         exchange.publish message, options
       end
     end
     
     def enqueue_and_wait message, options = {}
       result = nil
-      with_connection do
+      with_rpc_channel do |channel|
         correlation_id = options[:correlation_id] ||= generate_uuid
-        reply_queue = reply_queue correlation_id
+        reply_queue = reply_queue channel, correlation_id
         options[:reply_to] = reply_queue.name
         reply_queue.subscribe(:block => false) do |delivery_info, meta, payload|
           if meta[:correlation_id] == correlation_id
@@ -25,7 +26,8 @@ module Liebre
             channel.consumers[delivery_info.consumer_tag].cancel
           end
         end
-        logger.debug "Liebre: Publishing '#{message}' with '#{options}' to exchange: #{exchange}"
+        exchange = exchange_for channel
+        logger.debug "Liebre: Publishing '#{message}' with '#{options}' to exchange: #{exchange.name}"
         exchange.publish message, options
         begin
           Timeout.timeout(Liebre.config.rpc_request_timeout) do
@@ -33,12 +35,6 @@ module Liebre
           end
         rescue Timeout::Error
           #do nothing
-        ensure
-          begin
-            reply_queue.delete
-          rescue Timeout::Error
-            logger.error "error while trying to delete RPC exclusive queue"
-          end
         end
       end
       result
@@ -50,20 +46,34 @@ module Liebre
     
     def with_connection
       connection_manager.ensure_started
-      yield
+      begin
+        yield
+      rescue Bunny::Exception => e
+        logger.warn("#{self.class.name}: #{e.class} found restarting connection - #{e.message}")
+        connection_manager.restart
+        retry
+      end
     end
     
-    def reply_queue correlation_id
-      queue_name = "#{publisher_name}_callback_#{correlation_id}"
-      channel.queue queue_name, :exclusive => true
+    def with_rpc_channel
+      with_connection do
+        channel = connection_manager.get(connection_name).create_channel
+        yield(channel)
+        channel.close
+      end
+    end
+    
+    def default_channel
+      @default_channel ||= connection_manager.channel_for(connection_name)
     end
     
-    def exchange
-      @exchange ||= Liebre::Common::Utils.create_exchange channel, exchange_config
+    def reply_queue channel, correlation_id
+      queue_name = "#{publisher_name}_callback_#{correlation_id}"
+      channel.queue queue_name, :exclusive => true, :auto_delete => true
     end
     
-    def channel
-      @channel ||= connection_manager.get(connection_name).create_channel
+    def exchange_for channel 
+      Liebre::Common::Utils.create_exchange channel, exchange_config
     end
     
     def publishers
@@ -83,7 +93,7 @@ module Liebre
     end
 
     def connection_manager
-      @connection_manager ||= ConnectionManager.new
+      @connection_manager ||= ConnectionManager.instance
     end
     
     def generate_uuid

data/lib/liebre/runner.rb

@@ -12,7 +12,7 @@ module Liebre
 
     def start
       setup_shutdown
-      conn_manager.restart
+      connection_manager.restart
       start_consumers
       sleep
     rescue StandardError => e
@@ -29,13 +29,13 @@ module Liebre
     def do_shutdown
       Thread.start do
         logger.info("Liebre: Closing AMQP connection...")
-        conn_manager.stop
+        connection_manager.stop
         logger.info("Liebre: AMQP connection closed")
       end.join
     end
 
     def start_consumers
-      consumers = Consumers.new(conn_manager)
+      consumers = Consumers.new(connection_manager)
       consumers.start_all
     end
 
@@ -49,8 +49,8 @@ module Liebre
       Liebre.logger
     end
 
-    def conn_manager
-      @conn_manager ||= ConnectionManager.new
+    def connection_manager
+      @connection_manager ||= ConnectionManager.instance
     end
 
     attr_reader :retry_interval

data/lib/liebre/runner/starter/consumer.rb

@@ -2,7 +2,7 @@ module Liebre
   class Runner
     class Starter
       class Consumer
-        
+
         autoload :Handler,   "liebre/runner/starter/consumer/handler"
 
         def initialize connection, config
@@ -14,9 +14,9 @@ module Liebre
           initialize_error_queue
           initialize_queue
         end
-        
+
         private
-        
+
         def initialize_queue
           queue.subscribe(:manual_ack => true) do |info, meta, payload|
             response = :reject
@@ -34,19 +34,19 @@ module Liebre
             end
           end
         end
-        
+
         def initialize_error_queue
           Resources.new(connection, error_config).queue
         end
-        
+
         def klass
           @klass ||= Kernel.const_get config.fetch("class_name")
         end
-        
+
         def handler
           @handler ||= Handler.new(channel)
         end
-        
+
         def channel
           resources.channel
         end
@@ -62,32 +62,32 @@ module Liebre
         def resources
           @resources ||= Resources.new(connection, parse_config)
         end
-        
+
         def parse_config
           result = clone_hash config
           result['queue']['opts']['arguments'] ||= {}
           result['queue']['opts']['arguments']['x-dead-letter-exchange'] = result['exchange']['name'] + "-error"
           result
         end
-        
+
         def error_config
           result = clone_hash config
           result['exchange']['name'] += "-error"
           result['queue']['name'] += "-error"
           result
         end
-        
+
         def logger
           Liebre::Config.logger
         end
-        
+
         def clone_hash hash
           Marshal.load(Marshal.dump(hash))
         end
 
         attr_reader :connection, :config
-        
+
       end
     end
   end
-end
+end

data/lib/liebre/runner/starter/consumer/handler.rb

@@ -7,29 +7,29 @@ module Liebre
           def initialize channel
             @channel = channel
           end
-          
+
           def respond action, meta
             send(action, meta.delivery_tag)
           end
 
           private
-          
+
           def ack delivery_tag
             channel.acknowledge delivery_tag
           end
-          
+
           def reject delivery_tag
             channel.reject delivery_tag, true
           end
-          
+
           def error delivery_tag
-            channel.reject delivery_tag, false            
+            channel.reject delivery_tag, false
           end
 
           attr_reader :channel
-        
+
         end
       end
     end
   end
-end
+end

data/lib/liebre/runner/starter/resources.rb

@@ -18,15 +18,21 @@ module Liebre
         end
 
         def channel
-          @channel ||= connection.create_channel(nil, pool_size)
+          @channel ||= connection.create_channel(nil, pool_size).tap do |channel|
+            channel.prefetch(prefetch_count)
+          end
         end
 
         private
-        
+
         def queue_builder
           @queue_bilder ||= QueueBuilder.new(channel, config)
         end
 
+        def prefetch_count
+          config.fetch("prefetch_count", 10)
+        end
+
         def pool_size
           config.fetch("pool_size", 1)
         end
@@ -36,4 +42,4 @@ module Liebre
       end
     end
   end
-end
+end

data/lib/liebre/runner/starter/resources/queue_builder.rb

@@ -11,8 +11,12 @@ module Liebre
 
           def queue
             q = channel.queue(queue_name, queue_opts)
-            routing_keys.each do |key|
-              q.bind(exchange, bind_opts.merge(:routing_key => key))
+            if routing_keys.any?
+              routing_keys.each do |key|
+                q.bind(exchange, bind_opts.merge(:routing_key => key))
+              end
+            else
+              q.bind(exchange, bind_opts)
             end
             q
           end
@@ -40,8 +44,10 @@ module Liebre
           end
           
           def routing_keys
-            bind_opts[:routing_key] = [*bind_opts[:routing_key]]
-            bind_opts.delete :routing_key
+            @routing_keys ||= begin
+              bind_opts[:routing_key] = [*bind_opts[:routing_key]]
+              bind_opts.delete :routing_key
+            end
           end
 
           def bind_opts

data/lib/liebre/version.rb

@@ -1,5 +1,5 @@
 module Liebre
-  
-  VERSION = "0.1.5"
 
-end
+  VERSION = "0.1.12"
+
+end

data/spec/config/liebre.yml

@@ -6,6 +6,7 @@ consumers:
     rpc: false
     pool_size: 1
     num_threads: 3
+    prefetch_count: 5
     exchange:
       name: "consumer_exchange"
       type: "fanout"
@@ -15,7 +16,7 @@ consumers:
       name: "consumer_queue"
       opts:
         durable: false
-        
+
   some_rpc:
     class_name: MyRPC
     rpc: true
@@ -30,7 +31,7 @@ consumers:
       name: "rpc_queue"
       opts:
         durable: false
-  
+
 publishers:
   some_publisher:
     exchange:
@@ -44,4 +45,4 @@ publishers:
       name: "rpc_exchange"
       type: "fanout"
       opts:
-        durable: false
+        durable: false

data/spec/integration_spec.rb

@@ -1,73 +1,76 @@
 require 'spec_helper'
 
 RSpec.describe "Integration" do
-  
+
   class MyConsumer
-    
+
     def initialize payload, meta
       @payload = payload
       @meta = meta
     end
-    
+
   end
-  
+
   class MyRPC
-    
+
     def initialize payload, meta, callback
       @payload = payload
       @meta = meta
       @callback = callback
     end
-    
+
     def call
       @callback.call(@payload)
     end
-    
+
   end
-  
+
   let :config_path do
     File.expand_path("../config/liebre.yml" ,__FILE__)
   end
-    
+
   let :connection_path do
     File.expand_path("../config/rabbitmq.yml" ,__FILE__)
   end
-  
+
   before do
+    Liebre::ConnectionManager.instance_variable_set :@singleton__instance__, nil
     Liebre::Config.config_path     = config_path
     Liebre::Config.connection_path = connection_path
   end
-  
+
   let(:consumer) { double 'consumer' }
-  
+
   it do
-    
+
     main_thread = Thread.new do
       server = Liebre::Runner.new
       server.start
     end
-    
+
+    sleep 0.1
+
     publisher = Liebre::Publisher.new("some_publisher")
-    
+
     allow(MyConsumer).to receive(:new).with("hello", anything).and_return consumer
-    
+
     #the consumer returns first :ack, then :reject and the message gets requed, then :error, and the message turns dead-lettered
     expect(consumer).to receive(:call).and_return :ack, :reject, :error
-    
+
     publisher.enqueue "hello", :routing_key => "consumer_queue" #:ack
     publisher.enqueue "hello", :routing_key => "consumer_queue" #:reject then :error
-    
-    
-    
+
+
+
     rpc_publisher = Liebre::Publisher.new("rpc_publisher")
-    
+
     param = "return this string"
-    
+
     result = rpc_publisher.rpc param, :routing_key => "rpc_queue"
-    
+
     expect(result).to eq param
-    
+
     sleep 0.1
   end
-  
-end
+
+end

data/spec/liebre/config_spec.rb

@@ -1,15 +1,15 @@
 require 'spec_helper'
 
 RSpec.describe Liebre::Config do
-  
+
   let :config_path do
     File.expand_path("../../config/liebre.yml" ,__FILE__)
   end
-    
+
   let :connection_path do
     File.expand_path("../../config/rabbitmq.yml" ,__FILE__)
   end
-    
+
   before do
     described_class.config_path     = config_path
     described_class.connection_path = connection_path
@@ -26,37 +26,38 @@ RSpec.describe Liebre::Config do
       expect(described_class.env).to eq "some_env"
     end
   end
-  
+
   describe '#consumers' do
-    
+
     let(:consumer_names) { %w{ some_consumer some_rpc } }
-    
+
     let(:consumer_config) do
       {
         'class_name' => "MyConsumer",
         'rpc' => false
       }
     end
-    
+
     let(:rpc_config) do
       {
         'class_name' => "MyRPC",
         'rpc' => true
       }
     end
-    
+
     it do
-      
+
       expect(subject.consumers.keys).to eq consumer_names
-      
+
       expect(subject.consumers['some_consumer']['class_name']).to eq consumer_config['class_name']
       expect(subject.consumers['some_consumer']['rpc']).to eq consumer_config['rpc']
-      
+      expect(subject.consumers['some_consumer']['prefetch_count']).to eq 5
+
       expect(subject.consumers['some_rpc']['class_name']).to eq rpc_config['class_name']
       expect(subject.consumers['some_rpc']['rpc']).to eq rpc_config['rpc']
-      
+
     end
-    
+
   end
 
 end

data/spec/liebre/connection_manager_spec.rb

@@ -5,8 +5,12 @@ RSpec.describe Liebre::ConnectionManager do
   let :connection_path do
     File.expand_path("../../config/rabbitmq.yml" ,__FILE__)
   end
+  
+  before do
+    Liebre::Config.connection_path = connection_path
+  end
 
-  subject { described_class.new connection_path }
+  subject { described_class.instance }
 
   describe '.start and .get' do
 

data/spec/liebre/publisher_spec.rb

@@ -22,15 +22,14 @@ RSpec.describe Liebre::Publisher do
   end
   
   let(:channel)            { double 'channel' }
-  let(:bunny_connection)   { double 'bunny_conection',    :create_channel => channel }
-  let(:connection_manager) { double 'connection_manager', :get => bunny_connection }
-  let(:exchange)           { double 'exchange' }
+  let(:connection_manager) { double 'connection_manager' }
+  let(:exchange)           { double 'exchange', :name => "exchange" }
   
   before do
     allow_any_instance_of(Liebre::Config).to receive(:publishers).and_return publishers_config
     allow_any_instance_of(Liebre::Config).to receive(:rpc_request_timeout).and_return 10
     
-    expect(Liebre::ConnectionManager).to receive(:new).and_return connection_manager
+    allow(Liebre::ConnectionManager).to receive(:instance).and_return connection_manager
     expect(connection_manager).to receive(:ensure_started)
     
     expect(Liebre::Common::Utils).to receive(:create_exchange).
@@ -41,6 +40,10 @@ RSpec.describe Liebre::Publisher do
   subject { described_class.new 'test_publisher' }
   
   describe "#enqueue" do
+    before do
+      expect(connection_manager).to receive(:channel_for).with(:default).and_return channel
+    end
+    
     it do
       message = "abc"
       expect(exchange).to receive(:publish).with message, {}
@@ -56,11 +59,15 @@ RSpec.describe Liebre::Publisher do
     let(:reply_queue)      { double "reply_queue", :name => reply_queue_name }
     let(:delivery_info)    { double "delivery_info", :consumer_tag => "tag"}
     let(:consumer)         { double 'consumer' }
+    let(:connection)       { double 'connection' }
     let(:consumers)        { {"tag" => consumer} }
     
     before do
-      expect(channel).to receive(:queue).with(reply_queue_name, :exclusive => true).
+      expect(connection_manager).to receive(:get).with(:default).and_return connection
+      expect(connection).to receive(:create_channel).and_return channel
+      expect(channel).to receive(:queue).with(reply_queue_name, :exclusive => true, :auto_delete => true).
         and_return reply_queue
+      expect(channel).to receive(:close)
     end
     
     it do
@@ -75,8 +82,6 @@ RSpec.describe Liebre::Publisher do
         
         block.call(delivery_info, {:correlation_id => correlation_id}, answer)
       end
-      
-      expect(reply_queue).to receive(:delete)
     
       result = subject.enqueue_and_wait message, :correlation_id => correlation_id
       expect(result).to eq answer

data/spec/liebre/runner_spec.rb

@@ -1,32 +1,36 @@
 require 'spec_helper'
 
 RSpec.describe Liebre::Runner do
-
+    
+  let :connection_path do
+    File.expand_path("../../config/rabbitmq.yml" ,__FILE__)
+  end
 
   let(:interval) { 1234 }
 
   subject { described_class.new(interval) }
   let(:logger) { double 'logger' }
 
-  let(:conn_manager) { double 'conn_manager'}
+  let(:connection_manager) { double 'connection_manager'}
 
   let(:consumers) { double 'consumers' }
 
   before do
+    Liebre::Config.connection_path = connection_path
     
     allow(subject).to receive(:logger).
       and_return(logger)
 
-    allow(Liebre::ConnectionManager).to receive(:new).
-      and_return(conn_manager)
+    allow(Liebre::ConnectionManager).to receive(:instance).
+      and_return(connection_manager)
 
     allow(described_class::Consumers).to receive(:new).
-      with(conn_manager).and_return(consumers)
+      with(connection_manager).and_return(consumers)
   end
 
   describe '#run' do
     it 'logs and retries after fail' do
-      expect(conn_manager).to receive(:restart) do
+      expect(connection_manager).to receive(:restart) do
         raise "some error"
       end
 
@@ -38,7 +42,7 @@ RSpec.describe Liebre::Runner do
         expect(message).to match /Retrying/
       end
 
-      expect(conn_manager).to receive(:restart)
+      expect(connection_manager).to receive(:restart)
       expect(consumers   ).to receive(:start_all)
 
       expect(subject).to receive(:sleep)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: liebre
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.12
 platform: ruby
 authors:
 - jcabotc
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-27 00:00:00.000000000 Z
+date: 2017-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny