evrone-common-amqp 0.0.1

data/lib/evrone/common/amqp/supervisor/threaded.rb

@@ -0,0 +1,170 @@
+require 'thread'
+
+module Evrone
+  module Common
+    module AMQP
+      class Supervisor::Threaded
+
+        include Common::AMQP::Logger
+
+        POOL_INTERVAL = 0.5
+
+        Task = Struct.new(:object, :method, :id) do
+
+          attr_accessor :thread, :attempt, :start_at
+
+          def alive?
+            !!(thread && thread.alive?)
+          end
+
+          def inspect
+            %{#<Task
+                 object=#{object.to_s}
+                 method=#{method.inspect}
+                 id=#{id.inspect}
+                 alive=#{alive?}
+                 attempt=#{attempt}
+                 start_at=#{start_at}> }.gsub("\n", ' ').gsub(/ +/, ' ').strip
+          end
+        end
+
+        class SpawnAttemptsLimitReached < ::Exception ; end
+
+        class << self
+
+          @@shutdown = false
+
+          def build(tasks)
+            supervisor = new
+            tasks.each_pair do |k,v|
+              v.times do |n|
+                supervisor.add k, :subscribe, n
+              end
+            end
+            supervisor
+          end
+
+          def resume
+            @@shutdown = false
+          end
+
+          def shutdown?
+            @@shutdown
+          end
+
+          def shutdown
+            @@shutdown = true
+          end
+
+        end
+
+        def initialize
+          self.class.resume
+          @tasks = Array.new
+        end
+
+        def add(object, method, id)
+          @tasks.push Task.new(object, method, id).freeze
+        end
+
+        def size
+          @tasks.size
+        end
+
+        def shutdown?
+          self.class.shutdown?
+        end
+
+        def shutdown
+          self.class.shutdown
+        end
+
+        def run_async
+          Thread.new { run }.tap{|t| t.abort_on_exception = true }
+        end
+
+        def run
+          start_all_threads
+
+          loop do
+            task = @tasks.shift
+            break unless task
+
+            case
+            when shutdown?
+              log_thread_error task
+            when task.alive?
+              @tasks.push task
+            else
+              process_fail task
+            end
+
+            sleep POOL_INTERVAL unless shutdown?
+          end
+        end
+
+        private
+
+          def process_fail(task)
+            log_thread_error task
+            if check_attempt task
+              @tasks.push create_thread(task, task.attempt + 1)
+            else
+              raise SpawnAttemptsLimitReached
+            end
+          end
+
+          def start_all_threads
+            started_tasks = Array.new
+            while task = @tasks.shift
+              started_tasks.push create_thread(task, 0)
+            end
+            while task = started_tasks.shift
+              @tasks.push task
+            end
+          end
+
+          def create_thread(task, attempt)
+            attempt = 0 if reset_attempt?(task)
+            task.dup.tap do |new_task|
+              new_task.thread = Thread.new(new_task) do |t|
+                Thread.current[:consumer_id] = t.id
+                Thread.current[:consumer_name] = t.object.to_s
+                t.object.send t.method
+              end
+              new_task.thread.abort_on_exception = false
+              new_task.attempt = attempt
+              new_task.start_at = Time.now
+              new_task.freeze
+              debug "spawn #{new_task.inspect}"
+            end
+          end
+
+          def log_thread_error(task)
+            return unless task.thread
+
+            begin
+              task.thread.value
+              nil
+            rescue Exception => e
+              STDERR.puts "#{e.inspect} in #{task.inspect}"
+              STDERR.puts e.backtrace.join("\n")
+              e
+            end
+          end
+
+          def reset_attempt?(task)
+            return true unless task.start_at
+
+            interval = 60
+            (task.start_at + interval) < Time.now
+          end
+
+          def check_attempt(task)
+            task.attempt.to_i <= Common::AMQP.config.spawn_attempts.to_i
+          end
+
+      end
+    end
+  end
+end

data/lib/evrone/common/amqp/testing.rb

@@ -0,0 +1,46 @@
+require File.expand_path("../../amqp", __FILE__)
+
+module Evrone
+  module Common
+    module AMQP
+      module Testing
+
+        extend self
+
+        @@messages = []
+        @@exchange_messages = Hash.new { |h,k| h[k] = [] }
+
+        def messages
+          @@messages
+        end
+
+        def exchange_messages
+          @@exchange_messages
+        end
+
+        def clear
+          messages.clear
+          exchange_messages.clear
+        end
+      end
+
+      module Consumer::Publish
+        alias_method :real_publish, :publish
+
+        def publish(message, options = {})
+          Testing.exchange_messages[exchange_name] << message
+          Testing.messages << message
+          self
+        end
+      end
+
+      module Consumer
+        module ClassMethods
+          def messages
+            Testing.exchange_messages[exchange_name]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/evrone/common/amqp/version.rb

@@ -0,0 +1,7 @@
+module Evrone
+  module Common
+    module AMQP
+      VERSION = "0.0.1"
+    end
+  end
+end

data/spec/integration/multi_threaded_spec.rb

@@ -0,0 +1,83 @@
+require 'spec_helper'
+require 'thread'
+require 'timeout'
+
+class Evrone::BobThread
+  include Evrone::Common::AMQP::Consumer
+
+  queue    exclusive: true, durable: false
+  exchange auto_delete: true, durable: false
+  ack      true
+
+  def perform(payload)
+    $mtest_mutex.synchronize do
+      $mtest_collected << payload
+      ack!
+      sleep 0.1
+    end
+  end
+end
+
+class Evrone::AliceThread
+  include Evrone::Common::AMQP::Consumer
+
+  queue    exclusive: true, durable: false
+  exchange auto_delete: true, durable: false
+  ack      true
+
+  def perform(payload)
+    Evrone::BobThread.publish payload, content_type: properties[:content_type]
+    ack!
+    sleep 0.1
+  end
+end
+
+describe "Run in multithread environment", slow: true, jruby: true do
+  let(:num_messages) { 100 }
+  let(:alice)   { Evrone::AliceThread }
+  let(:bob)     { Evrone::BobThread }
+  let(:sess)    { Evrone::Common::AMQP.open }
+  let(:ch)      { sess.conn.create_channel }
+
+  before do
+    $mtest_mutex = Mutex.new
+    $mtest_collected = []
+  end
+
+  after do
+    sess.close
+  end
+
+  it "should be successfuly" do
+    ths = (0..12).map do |i|
+      klass = (i % 2 == 0) ? alice : bob
+      Thread.new do
+        klass.subscribe
+      end
+    end
+    ths.each{|t| t.abort_on_exception = true }
+    sleep 0.5
+
+    num_messages.times do |n|
+      alice.publish "n#{n}", content_type: "text/plain"
+    end
+
+    Timeout.timeout(60) do
+      loop do
+        stop = false
+        $mtest_mutex.synchronize do
+          puts $mtest_collected.size
+          stop = true if $mtest_collected.size >= num_messages
+        end
+        break if stop
+        sleep 2
+      end
+    end
+
+    Evrone::Common::AMQP.shutdown
+    Timeout.timeout(10) { ths.map{|i| i.join } }
+
+    expect($mtest_collected.sort).to eq (0...num_messages).map{|i| "n#{i}" }.sort
+  end
+
+end

data/spec/integration/threaded_supervisor_spec.rb

@@ -0,0 +1,85 @@
+require 'spec_helper'
+require 'thread'
+require 'timeout'
+
+class Evrone::BobThreadWithSupervisor
+  include Evrone::Common::AMQP::Consumer
+
+  class ErrorSimulation < ::Exception ; end
+
+  queue    exclusive: true, durable: false
+  exchange auto_delete: true, durable: false
+  ack      true
+
+  def perform(payload)
+    $mtest_mutex.synchronize do
+      raise IgnoreMeError if Random.new(delivery_info.delivery_tag.to_i).rand < 0.2
+      $mtest_collected << payload
+      ack!
+      sleep 0.1
+    end
+  end
+end
+
+class Evrone::AliceThreadWithSupervisor
+  include Evrone::Common::AMQP::Consumer
+
+  queue    exclusive: true, durable: false
+  exchange auto_delete: true, durable: false
+  ack      true
+
+  def perform(payload)
+    Evrone::BobThreadWithSupervisor.publish payload, content_type: properties[:content_type]
+    ack!
+    sleep 0.1
+  end
+end
+
+describe "Run in multithread environment", slow: true, jruby: true do
+  let(:num_messages) { 100 }
+  let(:alice)   { Evrone::AliceThreadWithSupervisor }
+  let(:bob)     { Evrone::BobThreadWithSupervisor }
+  let(:sess)    { Evrone::Common::AMQP.open }
+  let(:ch)      { sess.conn.create_channel }
+
+  before do
+    $mtest_mutex = Mutex.new
+    $mtest_collected = []
+  end
+
+  after do
+    sess.close
+  end
+
+  it "should be successfuly" do
+
+    supervisor = Evrone::Common::AMQP::Supervisor::Threaded.build alice => 6, bob => 6
+
+    supervisor_thread = supervisor.run_async
+
+    sleep 0.5
+
+    num_messages.times do |n|
+      alice.publish "n#{n}", content_type: "text/plain"
+    end
+
+    Timeout.timeout(60) do
+      loop do
+        stop = false
+        $mtest_mutex.synchronize do
+          puts $mtest_collected.size
+          stop = true if $mtest_collected.size >= num_messages
+        end
+        break if stop
+        sleep 2
+      end
+    end
+
+    Evrone::Common::AMQP.shutdown
+    supervisor.shutdown
+    Timeout.timeout(10) { supervisor_thread.join }
+
+    expect($mtest_collected.sort).to eq (0...num_messages).map{|i| "n#{i}" }.sort
+  end
+
+end

data/spec/lib/amqp/consumer_spec.rb

@@ -0,0 +1,281 @@
+require 'spec_helper'
+require 'timeout'
+require 'json'
+
+class Evrone::TestConsumer
+  include Evrone::Common::AMQP::Consumer
+
+  ack true
+
+  def perform(payload)
+    Thread.current[:collected] ||= []
+    Thread.current[:collected] << payload
+    ack!
+
+    :shutdown if Thread.current[:collected].size == 3
+  end
+end
+
+describe Evrone::Common::AMQP::Consumer do
+
+  let(:consumer) { Evrone::TestConsumer.new }
+  let(:consumer_class) { consumer.class }
+
+  subject { consumer }
+
+  before { consumer_class.reset_consumer_configuration! }
+
+  context '(configuration)' do
+
+    subject { consumer_class }
+
+    its(:consumer_name) { should eq 'evrone.test' }
+    its(:config)        { should be_an_instance_of(Evrone::Common::AMQP::Config) }
+
+    context "model" do
+      subject { consumer_class.model }
+
+      it "by default should be nil" do
+        expect(subject).to be_nil
+      end
+
+      it 'when set model should be' do
+        consumer_class.model Hash
+        expect(subject).to eq Hash
+      end
+    end
+
+    context "content_type" do
+
+      subject { consumer_class.content_type }
+
+      it "by default should be nil" do
+        expect(subject).to be_nil
+      end
+
+      it 'when set content type should be' do
+        consumer_class.content_type 'foo'
+        expect(subject).to eq 'foo'
+      end
+    end
+
+    context "bind_options" do
+      subject { consumer_class.bind_options }
+
+      context "by default should eq {}" do
+        it { should eq ({}) }
+      end
+
+      context "set routing_key" do
+        before { consumer_class.routing_key 'key' }
+        it { should eq({routing_key: 'key'}) }
+      end
+
+      context "set headers" do
+        before { consumer_class.headers 'key' }
+        it { should eq({headers: 'key'}) }
+      end
+    end
+
+    context "ack" do
+      subject { consumer_class.ack }
+
+      it "by default should be false" do
+        expect(subject).to be_false
+      end
+
+      it "when set to true should be true" do
+        consumer_class.ack true
+        expect(subject).to be_true
+      end
+    end
+
+    context "exchange_name" do
+      subject { consumer_class.exchange_name }
+
+      it 'by default should eq consumer_name' do
+        expect(subject).to eq consumer_class.consumer_name
+      end
+
+      it "when set name should be" do
+        consumer_class.exchange :foo
+        expect(subject).to eq :foo
+      end
+    end
+
+    context "queue_name" do
+      subject{ consumer_class.queue_name }
+      it 'by default should eq consumer_name' do
+        expect(subject).to eq consumer_class.consumer_name
+      end
+
+      it "when set name should be" do
+        consumer_class.queue :bar
+        expect(subject).to eq :bar
+      end
+    end
+
+    %w{ queue exchange }.each do |m|
+      context "#{m}_options" do
+        subject { consumer_class.send "#{m}_options" }
+        it 'by default should eq {}' do
+          expect(subject).to eq({})
+        end
+
+        it "when set #{m} options should be" do
+          consumer_class.send(m, durable: true)
+          expect(subject).to eq(durable: true)
+        end
+      end
+    end
+
+    %w{ routing_key headers }.each do |m|
+      context m do
+        subject { consumer_class.send m }
+
+        it 'by default should be nil' do
+          expect(subject).to be_nil
+        end
+
+        it "when set #{m} should be" do
+          consumer_class.send(m, key: :value)
+          expect(subject).to eq(key: :value)
+        end
+      end
+    end
+  end
+
+  context "(publish)" do
+
+    context "options" do
+      let(:message)          { {"foo" => 1, "bar" => 2} }
+      let(:expected_options) { {} }
+      let(:options)          { {} }
+      let(:x)                { OpenStruct.new name: "name" }
+
+      subject{ consumer_class.publish message, options }
+
+      before do
+        mock(consumer_class).declare_exchange { x }
+        mock(x).publish(message.to_json, expected_options)
+      end
+
+      context "routing_key" do
+        context "by default" do
+          it { should be }
+        end
+
+        context "when exists in configuration" do
+          let(:expected_options) { { routing_key: 'routing.key' } }
+          before do
+            consumer_class.routing_key 'routing.key'
+          end
+          it { should be }
+        end
+
+        context "when exists in options" do
+          let(:expected_options) { { routing_key: 'routing.key' } }
+          let(:options)          { { routing_key: 'routing.key' } }
+          it { should be }
+        end
+
+        context "when exists in options and configuration" do
+          let(:expected_options) { { routing_key: 'options.key' } }
+          let(:options)          { { routing_key: 'options.key' } }
+          before do
+            consumer_class.routing_key 'configuration.key'
+          end
+          it { should be }
+        end
+      end
+
+      context "headers" do
+        context "by default" do
+          it { should be }
+        end
+
+        context "when exists in configuration" do
+          let(:expected_options) { { headers: 'key' } }
+          before do
+            consumer_class.headers 'key'
+          end
+          it { should be }
+        end
+
+        context "when exists in options" do
+          let(:expected_options) { { headers: 'key' } }
+          let(:options)          { { headers: 'key' } }
+          it { should be }
+        end
+
+        context "when exists in options and configuration" do
+          let(:expected_options) { { headers: 'options' } }
+          let(:options)          { { headers: 'options' } }
+          before do
+            consumer_class.headers 'configuration'
+          end
+          it { should be }
+        end
+      end
+    end
+
+    context "real run" do
+      let(:x_name)  { consumer_class.exchange_name      }
+      let(:q_name)  { consumer_class.queue_name         }
+      let(:sess)    { consumer_class.session.open       }
+      let(:ch)      { sess.conn.create_channel          }
+      let(:q)       { sess.declare_queue q_name, channel: ch    }
+      let(:x)       { sess.declare_exchange x_name, channel: ch }
+      let(:message) { { 'key' => 'value' } }
+
+      after do
+        delete_queue q
+        delete_exchange x
+        sess.close
+      end
+
+      before do
+        q.bind x
+      end
+
+      it "should publish message to exchange using settings from consumer" do
+        consumer_class.publish message
+        sleep 0.25
+        expect(q.message_count).to eq 1
+        _, _, expected = q.pop
+        expect(expected).to eq message.to_json
+      end
+    end
+  end
+
+  context '(subscribe)' do
+    let(:x_name)  { consumer_class.exchange_name      }
+    let(:q_name)  { consumer_class.queue_name         }
+    let(:sess)    { consumer_class.session.open       }
+    let(:ch)      { sess.conn.create_channel          }
+    let(:q)       { sess.declare_queue q_name, channel: ch    }
+    let(:x)       { sess.declare_exchange x_name, channel: ch }
+
+    after do
+      delete_queue q
+      delete_exchange x
+      sess.close
+    end
+
+    before do
+      consumer_class.ack true
+      q.bind(x)
+      3.times { |n| x.publish({"n" => n}.to_json, content_type: "application/json") }
+    end
+
+    subject { Thread.current[:collected] }
+
+    it "should receive messages" do
+      Timeout.timeout(3) do
+        consumer_class.subscribe
+      end
+      expect(subject).to have(3).items
+      expect(subject.map(&:values).flatten).to eq [0,1,2]
+    end
+  end
+end