empipelines 0.2.3 → 0.2.4

data/README.md

@@ -1,11 +1,22 @@
-# EM Pipelines
+# EmPipelines
+
+## Message States
+
+* **Consumed:** The message went through all relevant stages and was
+fully consumed.
+* **Broken:** The message *left the EventSource* in an invalid
+state and should not be processed. If the message was corrupted by a
+previous stage it should not be broken, just rejected.
+* **Rejected:** The message is correct but this pipeline instance
+cannot process it, probably due to some temporary problem.
 
 ## TODO
 * Make wiring easier
 * Example apps
-* Control flow for AmqpEventSource
 * Evented I/O for IOEventSource
 * Consolidate logger and monitoring
 * Default monitoring implementation
 * Performance testing
 * Detect insonsistency when handler didnt consume or pass message ahead
+* Is callback in the stage's signature redundant? Could we just return
+the message?

data/empipelines.gemspec

@@ -4,8 +4,8 @@ Gem::Specification.new do |s|
   s.rubygems_version = '1.3.5'
   ## Leave these as is they will be modified for you by the rake gemspec task.
   s.name              = 'empipelines'
-  s.version           = '0.2.3'
-  s.date              = '2012-01-02'
+  s.version           = '0.2.4'
+  s.date              = '2012-01-10'
   s.rubyforge_project = 'empipelines'
 
   s.summary           = "Simple Event Handling Pipeline Architecture for EventMachine"
@@ -35,10 +35,12 @@ Gem::Specification.new do |s|
     Gemfile.lock
     README.md
     Rakefile
+    empipelines-0.2.4.gem
     empipelines.gemspec
+    functional/consuming_events_from_amqp_spec.rb
+    functional/consuming_events_from_batch_spec.rb
     functional/consuming_events_from_file_spec.rb
     functional/consuming_events_from_multiple_sources_spec.rb
-    functional/consuming_events_from_queue_spec.rb
     functional/events.dat
     functional/test_stages.rb
     lib/empipelines.rb

data/functional/consuming_events_from_amqp_spec.rb

@@ -0,0 +1,47 @@
+require 'eventmachine'
+require 'empipelines'
+require 'amqp'
+require 'json'
+require File.join(File.dirname(__FILE__), 'test_stages')
+
+module TestStages
+  ExchangeName = 'empipelines.build'
+  QueueName = 'empipelines.build.queue'
+
+  describe 'Consumption of events from a in-memory batch' do
+    let(:monitoring) { stub(:inform => nil, :debug => nil) }
+    let (:processed) { [] }
+
+    include EmRunner
+
+    it 'consumes all events from a queue' do
+      with_em_run do
+        connection = AMQP.connect()
+        channel = AMQP::Channel.new(connection)
+        channel.prefetch(1)
+
+        exchange = channel.direct(ExchangeName, :durable => true)
+        queue = channel.queue(QueueName, :durable => true)
+        queue.bind(exchange)
+        queue.purge
+
+        messages = (1..1000).map { |i| {:data => i}.to_json }
+        messages.each { |m| exchange.publish(m, :rounting_key => QueueName) }
+
+        pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring)
+        source = EmPipelines::AmqpEventSource.new(EM, queue, 'msg')
+
+        stages = [PassthroughStage, PassthroughStage, PassthroughStage]
+        event_pipeline = EmPipelines::EventPipeline.new(source, pipeline.for(stages), monitoring)
+
+        EM.add_periodic_timer(0.1) do
+          if processed.size == messages.size * stages.size then
+            EM.stop
+          end
+        end
+
+        event_pipeline.start!
+      end
+    end
+  end
+end

data/functional/{consuming_events_from_queue_spec.rb → consuming_events_from_batch_spec.rb}

@@ -3,15 +3,14 @@ require 'empipelines'
 require File.join(File.dirname(__FILE__), 'test_stages')
 
 module TestStages
-  describe 'Consumption of events from a queue' do
-    let(:monitoring) { stub() }
-    let(:logger) { stub(:info => nil, :debug => nil) }
-    let (:processed) { {} }
+  describe 'Consumption of events from a in-memory batch' do
+    let(:monitoring) { stub(:inform => nil, :debug => nil) }
+    let (:processed) { [] }
     include EmRunner
 
     it 'consumes all events from the a queue' do
       with_em_run do
-        pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring, logger)
+        pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring)
 
         batch = (1...1000).to_a
         batch_name = "my batch!"

data/functional/consuming_events_from_file_spec.rb

@@ -4,14 +4,13 @@ require File.join(File.dirname(__FILE__), 'test_stages')
 
 module TestStages
   describe 'Consumption of events from a file' do
-    let(:monitoring) { stub() }
-    let(:logger) { stub(:info => nil, :debug => nil) }
-    let (:processed) { {} }
+    let(:monitoring) { stub(:inform => nil, :debug => nil) }
+    let (:processed) { [] }
     include EmRunner
 
     it 'consumes all events from the file' do
       with_em_run do
-        pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring, logger)
+        pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring)
 
         file_name = File.join(File.dirname(__FILE__), 'events.dat')
         source = EmPipelines::IOEventSource.new(EM, file_name)

data/functional/consuming_events_from_multiple_sources_spec.rb

@@ -4,13 +4,12 @@ require File.join(File.dirname(__FILE__), 'test_stages')
 
 module TestStages
   describe 'Consumption of events from multiple sources' do
-    let(:monitoring) { stub() }
-    let(:logger) { stub(:info => nil, :debug => nil) }
-    let (:processed) { {} }
+    let(:monitoring) { stub(:inform => nil, :debug => nil) }
+    let (:processed) { [] }
     include EmRunner
 
     it 'consumes all events from all sources' do
-      pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring, logger)
+      pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring)
 
       file_name = File.join(File.dirname(__FILE__), 'events.dat')
       num_events_on_file = IO.readlines(file_name).size

data/functional/test_stages.rb

@@ -20,7 +20,7 @@ module TestStages
     end
     
     def call(message, &callback)
-      processed[self.class] = message
+      processed << [@id, message.co_id]
       process(message, callback)
     end          
   end

data/lib/empipelines/aggregated_event_source.rb

@@ -2,7 +2,7 @@ require 'empipelines/event_source'
 
 module EmPipelines
   class AggregatedEventSource <  EventSource
-    
+
     def initialize(em, *event_sources)
       @em, @sources = em, event_sources.flatten
     end
@@ -11,13 +11,14 @@ module EmPipelines
       finished = 0
       @sources.each do |s|
         s.on_event(event_handler)
-        
+
         s.on_finished do |*ignored|
           finished += 1
           finished! if finished == @sources.size
         end
+
         @em.next_tick { s.start! }
-      end      
+      end
     end
   end
 end

data/lib/empipelines/amqp_event_source.rb

@@ -12,14 +12,16 @@ module EmPipelines
     end
 
     def start!
-      @queue.subscribe do |header, json_payload|
+      @queue.subscribe(:ack => true) do |header, json_payload|
         message = Message.new({
-                                :header     => header,
                                 :origin     => @queue.name,
                                 :payload    => JSON.parse(json_payload),
                                 :event      => @event_name,
                                 :started_at => Time.now.to_i
                               })
+        message.on_consumed { |m| header.ack }
+        message.on_broken   { |m| header.reject(:requeue => true) }
+        message.on_rejected { |m| header.reject(:requeue => true) }
         event!(message)
       end
     end

data/lib/empipelines/pipeline.rb

@@ -6,9 +6,8 @@ module EmPipelines
       end
     end
     
-    def initialize(em, context, monitoring, logger)
+    def initialize(em, context, monitoring)
       @em = em
-      @logger = logger
       @context = context
       @monitoring = monitoring
     end
@@ -17,12 +16,11 @@ module EmPipelines
       stages = event_definition.map(&instantiate_with_dependencies)
 
       monitoring = @monitoring
-      logger = @logger
       
       first_stage_process = stages.reverse.reduce(TerminatorStage) do |current_head, next_stage|
         @em.spawn do |input|
           begin
-            logger.debug "#{next_stage.class}#notify with #{input}}"
+            monitoring.debug "#{next_stage.class}#notify with #{input}}"
             next_stage.call(input) do |output|
               current_head.notify(output)
             end
@@ -32,7 +30,7 @@ module EmPipelines
         end        
       end
       
-      @logger.info "Pipeline for event_definition is: #{stages.map(&:class).join('->')}"    
+      @monitoring.inform "Pipeline for event_definition is: #{stages.map(&:class).join('->')}"    
       first_stage_process
     end
     

data/lib/empipelines.rb

@@ -1,5 +1,5 @@
 module EmPipelines
-  VERSION = '0.2.3'
+  VERSION = '0.2.4'
 end
 
 require 'empipelines/message'

data/unit/empipelines/amqp_event_source_spec.rb

@@ -3,7 +3,7 @@ require 'empipelines/amqp_event_source'
 module EmPipelines
   class StubQueue
     attr_accessor :name
-    def subscribe(&code)
+    def subscribe(ack, &code)
       @code = code
     end
 
@@ -11,7 +11,7 @@ module EmPipelines
       @code.call(header,  payload)
     end
   end
-  
+
   describe AmqpEventSource do
     let (:em) { mock('eventmachine') }
 
@@ -23,7 +23,7 @@ module EmPipelines
       header = stub('header')
 
       received_messages = []
-      
+
       amqp_source = AmqpEventSource.new(em, queue, event_type)
       amqp_source.on_event { |e| received_messages << e }
       amqp_source.start!
@@ -32,10 +32,48 @@ module EmPipelines
 
       received_messages.size.should eql(1)
       received_messages.first[:origin].should ==(queue.name)
-      received_messages.first[:header].should ==(header)
       received_messages.first[:payload].should ==({:key => "value"})
       received_messages.first[:event].should ==(event_type)
       received_messages.first[:started_at].should_not be_nil
     end
+
+    it 'acknowledges consumed messages' do
+      queue = StubQueue.new
+      header = mock('header')
+
+      header.should_receive(:ack)
+
+      amqp_source = AmqpEventSource.new(em, queue, 'event type')
+      amqp_source.on_event { |e| e.consumed! }
+      amqp_source.start!
+
+      queue.publish(header, '{"key":"value"}')
+    end
+
+    it 'rejects  broken messages with no requeue' do
+      queue = StubQueue.new
+      header = mock('header')
+
+      header.should_receive(:reject)
+
+      amqp_source = AmqpEventSource.new(em, queue, 'event type')
+      amqp_source.on_event { |e| e.broken! }
+      amqp_source.start!
+
+      queue.publish(header, '{"key":"value"}')
+    end
+
+    it 'rejects rejected messages with requeue' do
+      queue = StubQueue.new
+      header = mock('header')
+
+      header.should_receive(:reject).with({:requeue => true})
+
+      amqp_source = AmqpEventSource.new(em, queue, 'event type')
+      amqp_source.on_event { |e| e.rejected! }
+      amqp_source.start!
+
+      queue.publish(header, '{"key":"value"}')
+    end
   end
 end

data/unit/empipelines/pipeline_spec.rb

@@ -96,14 +96,14 @@ module EmPipelines
   end
 
   describe Pipeline do
-    let(:logger) {stub(:info => true, :debug => true)}
+    let(:monitoring) { stub(:inform => nil, :debug => nil) }
 
     it 'chains the actions using processes' do
       event_chain = [AddOne, SquareIt, GlobalHolder]
       a_msg = msg({:data =>1})
       a_msg.should_receive(:consumed!)
 
-      pipelines = Pipeline.new(StubSpawner.new, {}, stub('monitoring'), logger)
+      pipelines = Pipeline.new(StubSpawner.new, {}, monitoring)
       pipeline = pipelines.for(event_chain)
       pipeline.notify(a_msg)
 
@@ -112,7 +112,7 @@ module EmPipelines
 
     it 'does not send to the next if last returned nil' do
       event_chain = [AddOne, SquareIt, DeadEnd, GlobalHolder]
-      pipelines = Pipeline.new(StubSpawner.new, {}, stub('monitoring'), logger)
+      pipelines = Pipeline.new(StubSpawner.new, {}, monitoring)
       pipeline = pipelines.for(event_chain)
       pipeline.notify(msg({:data => 1}))
       GlobalHolder.held.should be_nil
@@ -120,7 +120,7 @@ module EmPipelines
 
     it 'makes all objects in the context object available to stages' do
       event_chain = [NeedsAnApple, NeedsAnOrange, GlobalHolder]
-      pipelines = Pipeline.new(StubSpawner.new, {:apple => :some_apple, :orange => :some_orange}, stub('monitoring'), logger)
+      pipelines = Pipeline.new(StubSpawner.new, {:apple => :some_apple, :orange => :some_orange}, monitoring)
       a_msg = msg({})
       a_msg.should_receive(:consumed!)
 
@@ -132,15 +132,14 @@ module EmPipelines
     end
 
     it 'sends exception to the proper handler' do
-      monitoring = mock()
       monitoring.should_receive(:inform_exception!)
-      pipeline = Pipeline.new(StubSpawner.new, {}, monitoring, logger)
+      pipeline = Pipeline.new(StubSpawner.new, {}, monitoring)
       pipeline.for([BrokenStage]).notify(msg({}))
     end
 
     it 'flags the message as consumed if goest through all stages' do
       event_chain = [Passthrough, Passthrough]
-      pipelines = Pipeline.new(StubSpawner.new, {}, stub('monitoring'), logger)
+      pipelines = Pipeline.new(StubSpawner.new, {}, monitoring)
       pipeline = pipelines.for(event_chain)
       a_msg = msg({:data => :whatevah})
       a_msg.should_receive(:consumed!)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: empipelines
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-01-02 00:00:00.000000000Z
+date: 2012-01-10 00:00:00.000000000Z
 dependencies: []
 description: Simple Event Handling Pipeline Architecture for EventMachine
 email: pcalcado+empipelines@gmail.com
@@ -23,10 +23,12 @@ files:
 - Gemfile.lock
 - README.md
 - Rakefile
+- empipelines-0.2.4.gem
 - empipelines.gemspec
+- functional/consuming_events_from_amqp_spec.rb
+- functional/consuming_events_from_batch_spec.rb
 - functional/consuming_events_from_file_spec.rb
 - functional/consuming_events_from_multiple_sources_spec.rb
-- functional/consuming_events_from_queue_spec.rb
 - functional/events.dat
 - functional/test_stages.rb
 - lib/empipelines.rb
@@ -67,7 +69,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2169333879126328078
+      hash: 4556679468386328917
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: