empipelines 0.2.4 → 0.2.5

data/empipelines.gemspec

@@ -4,8 +4,8 @@ Gem::Specification.new do |s|
   s.rubygems_version = '1.3.5'
   ## Leave these as is they will be modified for you by the rake gemspec task.
   s.name              = 'empipelines'
-  s.version           = '0.2.4'
-  s.date              = '2012-01-10'
+  s.version           = '0.2.5'
+  s.date              = '2012-01-13'
   s.rubyforge_project = 'empipelines'
 
   s.summary           = "Simple Event Handling Pipeline Architecture for EventMachine"
@@ -35,7 +35,6 @@ Gem::Specification.new do |s|
     Gemfile.lock
     README.md
     Rakefile
-    empipelines-0.2.4.gem
     empipelines.gemspec
     functional/consuming_events_from_amqp_spec.rb
     functional/consuming_events_from_batch_spec.rb

data/functional/consuming_events_from_amqp_spec.rb

@@ -4,32 +4,38 @@ require 'amqp'
 require 'json'
 require File.join(File.dirname(__FILE__), 'test_stages')
 
+ExchangeName = 'empipelines.build'
+QueueName = 'empipelines.build.queue'
+
+def setup_queues
+  connection = AMQP.connect()
+  channel = AMQP::Channel.new(connection)
+  channel.prefetch(1)
+
+  exchange = channel.direct(ExchangeName, :durable => true)
+  queue = channel.queue(QueueName, :durable => true)
+  queue.bind(exchange)
+  queue.purge
+  [exchange, queue]
+end
+
 module TestStages
-  ExchangeName = 'empipelines.build'
-  QueueName = 'empipelines.build.queue'
 
   describe 'Consumption of events from a in-memory batch' do
-    let(:monitoring) { stub(:inform => nil, :debug => nil) }
+    let(:monitoring) { Monitoring.new }
     let (:processed) { [] }
 
     include EmRunner
 
     it 'consumes all events from a queue' do
       with_em_run do
-        connection = AMQP.connect()
-        channel = AMQP::Channel.new(connection)
-        channel.prefetch(1)
-
-        exchange = channel.direct(ExchangeName, :durable => true)
-        queue = channel.queue(QueueName, :durable => true)
-        queue.bind(exchange)
-        queue.purge
+        exchange, queue = setup_queues
 
         messages = (1..1000).map { |i| {:data => i}.to_json }
         messages.each { |m| exchange.publish(m, :rounting_key => QueueName) }
 
         pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring)
-        source = EmPipelines::AmqpEventSource.new(EM, queue, 'msg')
+        source = EmPipelines::AmqpEventSource.new(EM, queue, 'msg', monitoring)
 
         stages = [PassthroughStage, PassthroughStage, PassthroughStage]
         event_pipeline = EmPipelines::EventPipeline.new(source, pipeline.for(stages), monitoring)
@@ -43,5 +49,52 @@ module TestStages
         event_pipeline.start!
       end
     end
+
+    it 'discards broken messages' do
+            with_em_run do
+        exchange, queue = setup_queues
+
+        messages = (1..1000).map { |i| {:data => i}.to_json }
+        messages.each { |m| exchange.publish(m, :rounting_key => QueueName) }
+
+        pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring)
+        source = EmPipelines::AmqpEventSource.new(EM, queue, 'msg', monitoring)
+
+        stages = [BrokenMessageStage, ShouldNotBeReachedStage]
+        event_pipeline = EmPipelines::EventPipeline.new(source, pipeline.for(stages), monitoring)
+
+        EM.add_periodic_timer(0.1) do
+          if processed.size == messages.size then
+            EM.stop
+          end
+        end
+
+        event_pipeline.start!
+      end
+    end
+
+    it 'retries messages if told to do so' do
+      with_em_run do
+        exchange, queue = setup_queues
+
+        messages = [ {:data => "b"}.to_json ]
+        messages.each { |m| exchange.publish(m, :rounting_key => QueueName) }
+
+        pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring)
+        source = EmPipelines::AmqpEventSource.new(EM, queue, 'msg', monitoring)
+
+        stages = [RetryOscillator]
+        event_pipeline = EmPipelines::EventPipeline.new(source, pipeline.for(stages), monitoring)
+
+        EM.add_periodic_timer(0.1) do
+          if processed.size == 2 then
+            EM.stop
+          end
+        end
+
+        event_pipeline.start!
+      end
+    end
+
   end
 end

data/functional/test_stages.rb

@@ -1,4 +1,22 @@
 module TestStages
+  class Monitoring
+    def initialize(output = false)
+      @output = output
+    end
+    
+    def inform(txt)
+      puts "#{Time.now.usec}  INFO: #{txt}" if @output
+    end
+
+    def debug(txt)
+      puts "#{Time.now.usec} DEBUG: #{txt}" if @output
+    end
+
+    def inform_exception!(exc, origin, extra = nil)
+      puts "#{Time.now.usec} ERROR: #{exc} at #{origin} - #{extra}" if @output
+    end
+  end
+  
   module EmRunner
     def with_em_run(&test_body)
       EM.run do
@@ -33,6 +51,15 @@ module TestStages
     end
   end
 
+  class RetryOscillator
+    include SomeStage
+
+    def process(message, callback)
+      @flip = !@flip || false
+      message.rejected! if @flip
+    end
+  end
+
   class ShouldNotBeReachedStage
     include SomeStage
     

data/lib/empipelines/amqp_event_source.rb

@@ -7,22 +7,27 @@ module EmPipelines
   #this must have a on_finished!
   class AmqpEventSource < EventSource
 
-    def initialize(em, queue, event_name)
-      @em, @queue, @event_name = em, queue, event_name
+    def initialize(em, queue, event_name, monitoring)
+      @em, @queue, @event_name, @monitoring = em, queue, event_name, monitoring
     end
 
     def start!
       @queue.subscribe(:ack => true) do |header, json_payload|
-        message = Message.new({
-                                :origin     => @queue.name,
-                                :payload    => JSON.parse(json_payload),
-                                :event      => @event_name,
-                                :started_at => Time.now.to_i
-                              })
-        message.on_consumed { |m| header.ack }
-        message.on_broken   { |m| header.reject(:requeue => true) }
-        message.on_rejected { |m| header.reject(:requeue => true) }
-        event!(message)
+        begin
+          message = Message.new({
+                                  :origin     => @queue.name,
+                                  :payload    => JSON.parse(json_payload),
+                                  :event      => @event_name,
+                                  :started_at => Time.now.to_i
+                                })
+          message.on_consumed { |m| header.ack }
+          message.on_broken   { |m| header.reject(:requeue => false) }
+          message.on_rejected { |m| header.reject(:requeue => true) }
+          event!(message)
+        rescue => exc
+          @monitoring.inform_exception!(exc, self, 'removing message from queue')
+          header.reject(:requeue => false)
+        end
       end
     end
   end

data/lib/empipelines/pipeline.rb

@@ -5,35 +5,35 @@ module EmPipelines
         message.consumed!
       end
     end
-    
+
     def initialize(em, context, monitoring)
       @em = em
       @context = context
       @monitoring = monitoring
     end
-    
+
     def for(event_definition)
       stages = event_definition.map(&instantiate_with_dependencies)
 
       monitoring = @monitoring
-      
+
       first_stage_process = stages.reverse.reduce(TerminatorStage) do |current_head, next_stage|
-        @em.spawn do |input|
+        @em.spawn do |input_message|
           begin
-            monitoring.debug "#{next_stage.class}#notify with #{input}}"
-            next_stage.call(input) do |output|
+            monitoring.debug "#{next_stage.class}#notify with #{input_message}}"
+            next_stage.call(input_message) do |output|
               current_head.notify(output)
             end
           rescue => exception
-            monitoring.inform_exception!(exception, next_stage)
+            monitoring.inform_exception!(exception, next_stage, "Message #{input_message} is broken")
+            input_message.broken!
           end
-        end        
+        end
       end
-      
-      @monitoring.inform "Pipeline for event_definition is: #{stages.map(&:class).join('->')}"    
+      @monitoring.inform "Pipeline for event_definition is: #{stages.map(&:class).join('->')}"
       first_stage_process
     end
-    
+
     private
     def instantiate_with_dependencies
       lambda do |stage_class|

data/lib/empipelines.rb

@@ -1,5 +1,5 @@
 module EmPipelines
-  VERSION = '0.2.4'
+  VERSION = '0.2.5'
 end
 
 require 'empipelines/message'

data/unit/empipelines/amqp_event_source_spec.rb

@@ -14,6 +14,7 @@ module EmPipelines
 
   describe AmqpEventSource do
     let (:em) { mock('eventmachine') }
+    let (:monitoring) { mock('monitoring') }
 
     it 'wraps each AMQP message and send to listeners' do
       json_payload = '{"key":"value"}'
@@ -24,7 +25,7 @@ module EmPipelines
 
       received_messages = []
 
-      amqp_source = AmqpEventSource.new(em, queue, event_type)
+      amqp_source = AmqpEventSource.new(em, queue, event_type, monitoring)
       amqp_source.on_event { |e| received_messages << e }
       amqp_source.start!
 
@@ -43,20 +44,34 @@ module EmPipelines
 
       header.should_receive(:ack)
 
-      amqp_source = AmqpEventSource.new(em, queue, 'event type')
+      amqp_source = AmqpEventSource.new(em, queue, 'event type', monitoring)
       amqp_source.on_event { |e| e.consumed! }
       amqp_source.start!
 
       queue.publish(header, '{"key":"value"}')
     end
 
-    it 'rejects  broken messages with no requeue' do
+    it 'marks message as broken if cannot be parsed' do
       queue = StubQueue.new
       header = mock('header')
 
-      header.should_receive(:reject)
+      header.should_receive(:reject).with({:requeue => false})
+      monitoring.should_receive(:inform_exception!)
 
-      amqp_source = AmqpEventSource.new(em, queue, 'event type')
+      amqp_source = AmqpEventSource.new(em, queue, 'event type', monitoring)
+      amqp_source.on_event { raise 'should never happen' }
+      amqp_source.start!
+  
+      queue.publish(header, 'some junk')
+    end
+    
+    it 'rejects broken messages with no requeue' do
+      queue = StubQueue.new
+      header = mock('header')
+
+      header.should_receive(:reject).with({:requeue => false})
+
+      amqp_source = AmqpEventSource.new(em, queue, 'event type', monitoring)
       amqp_source.on_event { |e| e.broken! }
       amqp_source.start!
 
@@ -69,7 +84,7 @@ module EmPipelines
 
       header.should_receive(:reject).with({:requeue => true})
 
-      amqp_source = AmqpEventSource.new(em, queue, 'event type')
+      amqp_source = AmqpEventSource.new(em, queue, 'event type', monitoring)
       amqp_source.on_event { |e| e.rejected! }
       amqp_source.start!
 

data/unit/empipelines/pipeline_spec.rb

@@ -10,7 +10,7 @@ module EmPipelines
     def consumed!
       raise 'unexpected call'
     end
-    
+
     def rejected!
       raise 'unexpected call'
     end
@@ -61,7 +61,7 @@ module EmPipelines
       next_stage.call(input.merge!({:orange => orange}))
     end
   end
-  
+
   class GlobalHolder
     @@value = nil
     def GlobalHolder.held
@@ -71,7 +71,7 @@ module EmPipelines
     def initialize
       @@value = nil
     end
-    
+
     def call(input, &next_step)
       @@value = input
       next_step.call(input)
@@ -79,17 +79,17 @@ module EmPipelines
   end
 
   class StubSpawner
-    
+
     class StubProcess
       def initialize(block)
         @block = block
       end
-      
+
       def notify(input)
         @block.call(input)
       end
     end
-    
+
     def spawn(&block)
       StubProcess.new(block)
     end
@@ -131,10 +131,14 @@ module EmPipelines
       GlobalHolder.held[:orange].should ==(:some_orange)
     end
 
-    it 'sends exception to the proper handler' do
+    it 'marks message as broken if uncaught exception' do
+      a_msg = msg({})
       monitoring.should_receive(:inform_exception!)
+
+      a_msg.should_receive(:broken!)
+
       pipeline = Pipeline.new(StubSpawner.new, {}, monitoring)
-      pipeline.for([BrokenStage]).notify(msg({}))
+      pipeline.for([BrokenStage]).notify(a_msg)
     end
 
     it 'flags the message as consumed if goest through all stages' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: empipelines
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-01-10 00:00:00.000000000Z
+date: 2012-01-13 00:00:00.000000000Z
 dependencies: []
 description: Simple Event Handling Pipeline Architecture for EventMachine
 email: pcalcado+empipelines@gmail.com
@@ -23,7 +23,6 @@ files:
 - Gemfile.lock
 - README.md
 - Rakefile
-- empipelines-0.2.4.gem
 - empipelines.gemspec
 - functional/consuming_events_from_amqp_spec.rb
 - functional/consuming_events_from_batch_spec.rb
@@ -69,7 +68,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4556679468386328917
+      hash: 3829933959180740476
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: