empipelines 0.2.1 → 0.2.2

data/README.md

@@ -3,7 +3,6 @@
 ## TODO
 * Make wiring easier
 * Example apps
-* TerminatorStage should consume message
 * Control flow for AmqpEventSource
 * Transaction ID on each message
 * Evented I/O for IOEventSource

data/Rakefile

@@ -116,6 +116,7 @@ task :release => :ci do
     git tag -a #{gem_file} -m 'Version #{gem_file}'
     git push origin master
     git push origin --tags
+    gem push pkg/#{gem_file}
   END
 end
 

data/empipelines.gemspec

@@ -4,8 +4,8 @@ Gem::Specification.new do |s|
   s.rubygems_version = '1.3.5'
   ## Leave these as is they will be modified for you by the rake gemspec task.
   s.name              = 'empipelines'
-  s.version           = '0.2.1'
-  s.date              = '2011-12-21'
+  s.version           = '0.2.2'
+  s.date              = '2011-12-23'
   s.rubyforge_project = 'empipelines'
 
   s.summary           = "Simple Event Handling Pipeline Architecture for EventMachine"
@@ -37,9 +37,12 @@ Gem::Specification.new do |s|
     Rakefile
     empipelines.gemspec
     functional/consuming_events_from_file_spec.rb
+    functional/consuming_events_from_multiple_sources_spec.rb
+    functional/consuming_events_from_queue_spec.rb
     functional/events.dat
     functional/test_stages.rb
     lib/empipelines.rb
+    lib/empipelines/aggregated_event_source.rb
     lib/empipelines/amqp_event_source.rb
     lib/empipelines/batch_event_source.rb
     lib/empipelines/event_handlers.rb
@@ -49,6 +52,7 @@ Gem::Specification.new do |s|
     lib/empipelines/message.rb
     lib/empipelines/periodic_event_source.rb
     lib/empipelines/pipeline.rb
+    unit/empipelines/aggregated_event_source_spec.rb
     unit/empipelines/amqp_event_source_spec.rb
     unit/empipelines/batch_event_source_spec.rb
     unit/empipelines/empty_io_event_source.dat

data/functional/consuming_events_from_file_spec.rb

@@ -10,29 +10,17 @@ module TestStages
     include EmRunner
 
     it 'consumes all events from the file' do
-      with_em_run do      
+      with_em_run do
         pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring, logger)
 
         file_name = File.join(File.dirname(__FILE__), 'events.dat')
         source = EmPipelines::IOEventSource.new(EM, file_name)
 
-        stages = [PassthroughStage, PassthroughStage, PassthroughStage, ConsumeStage]
+        stages = [PassthroughStage, PassthroughStage, PassthroughStage]
         event_pipeline = EmPipelines::EventPipeline.new(source, pipeline.for(stages), monitoring)
 
-        source.on_finished do |messages|
-          EM.stop  
-          messages.should have(10).items
-          messages[0][:payload].should ==("event #0")
-          messages[1][:payload].should ==("event #1")
-          messages[2][:payload].should ==("event #2")
-          messages[3][:payload].should ==("event #3")
-          messages[4][:payload].should ==("event #4")
-          messages[5][:payload].should ==("event #5")
-          messages[6][:payload].should ==("event #6")
-          messages[7][:payload].should ==("event #7")
-          messages[8][:payload].should ==("event #8")
-          messages[9][:payload].should ==("event #9")
-          messages.each { |m| m.state.should ==(:consumed) }        
+        source.on_finished do |s|
+          EM.stop          
         end
         
         event_pipeline.start!

data/functional/consuming_events_from_multiple_sources_spec.rb

@@ -0,0 +1,31 @@
+require 'eventmachine'
+require 'empipelines'
+require File.join(File.dirname(__FILE__), 'test_stages')
+
+module TestStages
+  describe 'Consumption of events from multiple sources' do
+    let(:monitoring) { stub() }
+    let(:logger) { stub(:info => nil, :debug => nil) }
+    let (:processed) { {} }
+    include EmRunner
+
+    it 'consumes all events from all sources' do
+      pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring, logger)
+
+      file_name = File.join(File.dirname(__FILE__), 'events.dat')
+      num_events_on_file = IO.readlines(file_name).size
+      io_source = EmPipelines::IOEventSource.new(EM, file_name)
+
+      batch = (1...1000).to_a
+      batch_name = "my batch!"
+      batch_source = EmPipelines::BatchEventSource.new(EM, batch_name, batch)
+
+      composed_event_source = EmPipelines::AggregatedEventSource.new(EM, batch_source, io_source)
+
+      composed_event_source.on_finished do |s|
+        EM.stop
+        s.should ==(composed_event_source)
+      end
+    end
+  end
+end

data/functional/consuming_events_from_queue_spec.rb

@@ -0,0 +1,32 @@
+require 'eventmachine'
+require 'empipelines'
+require File.join(File.dirname(__FILE__), 'test_stages')
+
+module TestStages
+  describe 'Consumption of events from a queue' do
+    let(:monitoring) { stub() }
+    let(:logger) { stub(:info => nil, :debug => nil) }
+    let (:processed) { {} }
+    include EmRunner
+
+    it 'consumes all events from the a queue' do
+      with_em_run do
+        pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring, logger)
+
+        batch = (1...1000).to_a
+        batch_name = "my batch!"
+        source = EmPipelines::BatchEventSource.new(EM, batch_name, batch)
+
+        stages = [PassthroughStage, PassthroughStage, PassthroughStage]
+        event_pipeline = EmPipelines::EventPipeline.new(source, pipeline.for(stages), monitoring)
+
+        source.on_finished do |s|
+          EM.stop
+          s.should ==(source)
+        end
+
+        event_pipeline.start!
+      end
+    end
+  end
+end

data/functional/events.dat

@@ -1,10 +1,3 @@
-event #0
-event #1
-event #2
-event #3
-event #4
-event #5
-event #6
-event #7
-event #8
-event #9
+1baa575127c661dfa754a43238b51bdb6adbc0ead0a3b1abec0d45e394ee19f1
+c9730aa998cda290da19a311d19bb91a6aefa3552795dfbd8a2083bf833ba645
+3fc87b82e170d9e01889537f1dee465e829461178b6eb0ea49b0429e4c75d798

data/functional/test_stages.rb

@@ -29,7 +29,7 @@ module TestStages
     include SomeStage
     
     def process(message, callback)
-      callback.call(message)
+      callback.call(message.merge!({}))
     end
   end
 

data/lib/empipelines/aggregated_event_source.rb

@@ -0,0 +1,24 @@
+require 'empipelines/event_handlers'
+
+module EmPipelines
+  class AggregatedEventSource
+    include EventHandlers
+    
+    def initialize(em, *event_sources)
+      @em, @sources = em, event_sources.flatten
+    end
+
+    def start!
+      finished = 0
+      @sources.each do |s|
+        s.on_event(event_handler)
+        
+        s.on_finished do |*ignored|
+          finished += 1
+          finished_handler.call(self) if finished == @sources.size
+        end
+        @em.next_tick { s.start! }
+      end      
+    end
+  end
+end

data/lib/empipelines/amqp_event_source.rb

@@ -1,6 +1,7 @@
 require 'json'
 
 module EmPipelines
+  #this must have a on_finished!
   class AmqpEventSource
     def initialize(em, queue, event_name)
       @em, @queue, @event_name = em, queue, event_name

data/lib/empipelines/batch_event_source.rb

@@ -5,6 +5,7 @@ module EmPipelines
     include EventHandlers
     
     def initialize(em, list_name, events)
+      @num_finalised = 0
       @em, @list_name, @events = em, list_name,events
     end
 
@@ -13,7 +14,7 @@ module EmPipelines
       check_if_finished
 
       message_finished = lambda do |m|
-        @finalised << m
+        @num_finalised += 1
         check_if_finished
       end
 
@@ -33,10 +34,13 @@ module EmPipelines
 
     private
     def check_if_finished
-      finished = (@finalised.size == @events.size)
+      #TODO: can we make this not be based on size?
+      #it makes it harder to have streams as event sources (i.e. ranges).
+      #this class should only rely on Enumerable methods.
+      finished = (@num_finalised == @events.size)
 
       if finished and finished_handler
-        @em.next_tick { finished_handler.call(@finalised) }
+        @em.next_tick { finished_handler.call(self) }
       end
     end
   end

data/lib/empipelines/io_event_source.rb

@@ -15,7 +15,7 @@ module EmPipelines
 
       wrapped_handler = BatchEventSource.new(@em, @file_path, events)
       wrapped_handler.on_event(event_handler)
-      wrapped_handler.on_finished(finished_handler)
+      wrapped_handler.on_finished { |*ignored| finished_handler.call(self) }
       wrapped_handler.start!
     end
   end

data/lib/empipelines/message.rb

@@ -3,27 +3,28 @@ module EmPipelines
     attr_reader :state
     
     def initialize(base_hash={})
-      hash!(base_hash)
+      backing_hash!(base_hash)
       created!
     end
 
     def [](key)
-      hash[key]
+      as_hash[key]
     end
 
     def []=(key, value)
       check_if_mutation_allowed
-      hash[key] = value
+      as_hash[key] = value
     end
 
     def delete(key)
       check_if_mutation_allowed
-      hash.delete key
+      as_hash.delete key
     end
 
-    def merge(other_hash)
+    def merge!(other_hash)
       check_if_mutation_allowed
-      Message.new(hash.merge(other_hash))
+      backing_hash!(as_hash.merge(other_hash))
+      self
     end
 
     def on_consumed(callback=nil, &callback_block)
@@ -56,17 +57,21 @@ module EmPipelines
       invoke(@rejected_broken_callback)
     end
 
-    def hash
+    def as_hash
       @backing_hash
     end
+
+    def payload
+      as_hash[:payload]
+    end
     
     def to_s
-      "#{self.class.name} state:#{@state} hash:#{hash}"
+      "#{self.class.name} state:#{@state} backing_hash:#{as_hash}"
     end
 
     private
 
-    def hash!(other)
+    def backing_hash!(other)
       @backing_hash = symbolised(other)
     end
     

data/lib/empipelines/periodic_event_source.rb

@@ -1,5 +1,6 @@
 module EmPipelines
   class PeriodicEventSource
+    #on finish!!!!
     def initialize(em, name, interval_in_secs, &event_sourcing_code)
       @em                  = em
       @name                = name

data/lib/empipelines/pipeline.rb

@@ -1,8 +1,8 @@
 module EmPipelines
   class Pipeline
     class TerminatorStage
-      def self.notify(ignored, also_ignored = {})
-        #noop
+      def self.notify(message, ignored = {})
+        message.consumed!
       end
     end
     

data/lib/empipelines.rb

@@ -1,13 +1,16 @@
 module EmPipelines
-  VERSION = '0.2.1'
+  VERSION = '0.2.2'
 end
 
+require 'empipelines/message'
+
 require 'empipelines/event_handlers'
 require 'empipelines/amqp_event_source'
 require 'empipelines/batch_event_source'
 require 'empipelines/io_event_source'
-require 'empipelines/event_pipeline'
 require 'empipelines/list_event_source'
-require 'empipelines/message'
 require 'empipelines/periodic_event_source'
+require 'empipelines/aggregated_event_source'
+
+require 'empipelines/event_pipeline'
 require 'empipelines/pipeline'

data/unit/empipelines/aggregated_event_source_spec.rb

@@ -0,0 +1,85 @@
+require 'empipelines/aggregated_event_source'
+
+module EmPipelines
+  class EventSourceStub
+    include EventHandlers
+
+    def event!(contents)
+      raise 'not started' unless @started
+      event_handler.call(contents) if event_handler
+    end
+
+    def finish!
+      raise 'not started' unless @started
+      finished_handler.call([:this, :should, :not, :be, :used])
+    end
+
+    def start!
+      @started = true
+    end
+  end
+
+  describe AggregatedEventSource do
+
+    let (:em) do
+      em = mock('eventmachine')
+      em.stub(:next_tick).and_yield
+      em
+    end
+
+    let (:list_name) { "list of stuff"  }
+
+    it 'sends each sends messages from all sources, as they happen, to listeners' do
+      source1, source2, source3 = EventSourceStub.new, EventSourceStub.new, EventSourceStub.new
+
+      aggregated = AggregatedEventSource.new(em, source1, source2, source3)
+
+      expected = (0..4).map{ |i| stub("Message #{i}") }
+      received = []
+      aggregated.on_event { |m| received << m}
+
+      aggregated.start!
+
+      source1.event! expected[0]
+      source2.event! expected[1]
+      source2.event! expected[2]
+      source3.event! expected[3]
+      source1.event! expected[4]
+
+      received.should ==(expected)
+    end
+
+    it 'calls the finished handler when all sources finished' do
+      sources = [EventSourceStub.new, EventSourceStub.new, EventSourceStub.new]
+
+      aggregated = AggregatedEventSource.new(em, sources)
+
+      has_finished = [false]
+      aggregated.on_finished do |s|
+        s.should ==(aggregated)
+        has_finished[0] = true
+      end
+
+      aggregated.start!
+      sources[2].finish!
+      sources[1].finish!
+      sources[0].finish!
+
+      has_finished.first.should be_true
+    end
+
+    it 'does not call the finished handler if a source is still going' do
+      sources = [EventSourceStub.new, EventSourceStub.new, EventSourceStub.new]
+
+      aggregated = AggregatedEventSource.new(em, sources)
+
+      aggregated.on_finished do |s|
+        raise 'should never happen'
+      end
+
+      aggregated.start!
+      sources[2].finish!
+      sources[0].finish!
+    end
+  end
+end

data/unit/empipelines/batch_event_source_spec.rb

@@ -1,6 +1,7 @@
 require 'empipelines/batch_event_source'
 
 module EmPipelines
+  ShouldNotBeCalled = lambda { raise 'should not be called' }
   describe BatchEventSource do
 
     let (:em) do
@@ -9,9 +10,9 @@ module EmPipelines
       em
     end
 
-    let (:list_name) { "list of stuff"  }
+    let (:list_name) { 'list of stuff'  }
 
-    it "sends each element on the list as a payload to the listener" do
+    it 'sends each element on the list as a payload to the listener' do
       events = [1,2,3,4,5,6,7,8,9,10]
       source = BatchEventSource.new(em, list_name, events)
 
@@ -26,14 +27,15 @@ module EmPipelines
       received.each{ |i| i[:origin].should == list_name }
     end
 
-    it "calls the batch finished callback when all items were processed" do
+    it 'calls the batch finished callback when all items were processed' do
       events = [1,2,3,4,5,6,7,8,9,10]
       source = BatchEventSource.new(em, list_name, events)
 
-      has_finished = []
+      has_finished = [false]
 
-      source.on_finished do |messages|
-        has_finished << messages
+      source.on_finished do |s|
+        s.should ==(source)
+        has_finished[0] = true
       end
 
       source.on_event do |e|
@@ -42,33 +44,30 @@ module EmPipelines
 
       source.start!
 
-      has_finished.first.map{ |i| i[:payload] }.should ==(events)
+      has_finished.first.should be_true
     end
 
-    it "finishes immediately if there are no events to process" do
+    it 'finishes immediately if there are no events to process' do
       source = BatchEventSource.new(em, list_name, [])
 
       has_finished = []
-      source.on_finished do |messages|
+      source.on_finished do |s|
+        s.should ==(source)
         has_finished << true
       end
 
-      source.on_event do |e|
-        raise 'should not be called!'
-      end
+      source.on_event(&ShouldNotBeCalled)
 
       source.start!
 
       has_finished.first.should be_true
     end
 
-    it "only calls the finished handler if all events were processed" do
+    it 'only calls the finished handler if all events were processed' do
       events = [1,2,3,4,5,6,7,8,9,10]
       source = BatchEventSource.new(em, list_name, events)
 
-      source.on_finished do |messages|
-        raise "should not be called"
-      end
+      source.on_event(&ShouldNotBeCalled)
 
       count = 0
       source.on_event do |e|

data/unit/empipelines/io_event_source_spec.rb

@@ -33,10 +33,11 @@ module EmPipelines
     it 'calls the finished callback when all messages were processed' do
       source = IOEventSource.new(em, events_file)
 
-      has_finished = []
+      has_finished = [false]
 
-      source.on_finished do |messages|
-        has_finished << messages
+      source.on_finished do |s|
+        s.should ==(source)
+        has_finished[0] = true
       end
 
       source.on_event do |e|
@@ -45,15 +46,16 @@ module EmPipelines
 
       source.start!
 
-      has_finished.first.map{ |i| i[:payload].to_i }.should ==([1,2,3])
+      has_finished.first.should be_true
     end
     
     it 'finishes immediately if there are no events to process' do
       source = IOEventSource.new(em, empty_file)
 
-      has_finished = []
-      source.on_finished do |messages|
-        has_finished << true
+      has_finished = [false]
+      source.on_finished do |s|
+        s.should ==(source)
+        has_finished[0] = true
       end
 
       source.on_event do |e|

data/unit/empipelines/message_spec.rb

@@ -46,12 +46,10 @@ module EmPipelines
 
       it "can be merged with a map, symbolising keys" do
         original = Message.new({'a' => 1})
-        merged = original.merge({'b' => 2})
+        original.merge!({'b' => 2})
 
-        original[:b].should be_nil
-        
-        merged[:a].should ==(original[:a])
-        merged[:b].should_not be_nil
+        original[:a].should ==(1)
+        original[:b].should ==(2)
       end
     end
 

data/unit/empipelines/pipeline_spec.rb

@@ -1,21 +1,45 @@
-require "empipelines/pipeline"
+require 'empipelines/pipeline'
+
+def msg(some_map)
+  EmPipelines:: MessageMock.new(some_map)
+end
 
 module EmPipelines
+  class MessageMock < EmPipelines::Message
+    def consumed!
+      raise 'unexpected call'
+    end
+    
+    def rejected!
+      raise 'unexpected call'
+    end
+
+    def broken!
+      raise 'unexpected call'
+    end
+  end
+
   class AddOne
     def call(input, &next_stage)
-      next_stage.call({:data => (input[:data] + 1)})
+      next_stage.call(input.merge!({:data => (input[:data] + 1)}))
+    end
+  end
+
+  class Passthrough
+    def call(input, &next_stage)
+      next_stage.call(input)
     end
   end
 
   class SquareIt
     def call(input, &next_stage)
-      next_stage.call({:data => (input[:data] * input[:data])})
+      next_stage.call(input.merge!({:data => (input[:data] * input[:data])}))
     end
   end
 
   class BrokenStage
     def call(ignore, &ignored_too)
-      raise "Boo!"
+      raise 'Boo!'
     end
   end
 
@@ -27,13 +51,13 @@ module EmPipelines
 
   class NeedsAnApple
     def call(input, &next_stage)
-      next_stage.call(input.merge({:apple => apple}))
+      next_stage.call(input.merge!({:apple => apple}))
     end
   end
 
   class NeedsAnOrange
     def call(input, &next_stage)
-      next_stage.call(input.merge({:orange => orange}))
+      next_stage.call(input.merge!({:orange => orange}))
     end
   end
   
@@ -49,7 +73,7 @@ module EmPipelines
     
     def call(input, &next_step)
       @@value = input
-      next_step.call(self.class)
+      next_step.call(input)
     end
   end
 
@@ -72,35 +96,55 @@ module EmPipelines
 
   describe Pipeline do
     let(:logger) {stub(:info => true, :debug => true)}
-    it "chains the actions using processes" do
+
+    it 'chains the actions using processes' do
       event_chain = [AddOne, SquareIt, GlobalHolder]
+      a_msg = msg({:data =>1})
+      a_msg.should_receive(:consumed!)
+
       pipelines = Pipeline.new(StubSpawner.new, {}, stub('monitoring'), logger)
       pipeline = pipelines.for(event_chain)
-      pipeline.notify({:data =>1})
-      GlobalHolder.held.should eql({:data => 4})
+      pipeline.notify(a_msg)
+
+      GlobalHolder.held[:data].should ==(4)
     end
 
-    it "does not send to the next if last returned nil" do
+    it 'does not send to the next if last returned nil' do
       event_chain = [AddOne, SquareIt, DeadEnd, GlobalHolder]
       pipelines = Pipeline.new(StubSpawner.new, {}, stub('monitoring'), logger)
       pipeline = pipelines.for(event_chain)
-      pipeline.notify({:data => 1})
+      pipeline.notify(msg({:data => 1}))
       GlobalHolder.held.should be_nil
     end
 
-    it "makes all objects in the context object available to stages" do
+    it 'makes all objects in the context object available to stages' do
       event_chain = [NeedsAnApple, NeedsAnOrange, GlobalHolder]
       pipelines = Pipeline.new(StubSpawner.new, {:apple => :some_apple, :orange => :some_orange}, stub('monitoring'), logger)
+      a_msg = msg({})
+      a_msg.should_receive(:consumed!)
+
       pipeline = pipelines.for(event_chain)
-      pipeline.notify({})
-      GlobalHolder.held.should eql({:apple => :some_apple, :orange => :some_orange})
+      pipeline.notify(a_msg)
+
+      GlobalHolder.held[:apple].should ==(:some_apple)
+      GlobalHolder.held[:orange].should ==(:some_orange)
     end
 
-    it "sends exception to the proper handler" do
+    it 'sends exception to the proper handler' do
       monitoring = mock()
       monitoring.should_receive(:inform_exception!)
       pipeline = Pipeline.new(StubSpawner.new, {}, monitoring, logger)
-      pipeline.for([BrokenStage]).notify({})
+      pipeline.for([BrokenStage]).notify(msg({}))
+    end
+
+    it 'flags the message as consumed if goest through all stages' do
+      event_chain = [Passthrough, Passthrough]
+      pipelines = Pipeline.new(StubSpawner.new, {}, stub('monitoring'), logger)
+      pipeline = pipelines.for(event_chain)
+      a_msg = msg({:data => :whatevah})
+      a_msg.should_receive(:consumed!)
+
+      pipeline.notify(a_msg)
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: empipelines
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-12-21 00:00:00.000000000Z
+date: 2011-12-23 00:00:00.000000000Z
 dependencies: []
 description: Simple Event Handling Pipeline Architecture for EventMachine
 email: pcalcado+empipelines@gmail.com
@@ -25,9 +25,12 @@ files:
 - Rakefile
 - empipelines.gemspec
 - functional/consuming_events_from_file_spec.rb
+- functional/consuming_events_from_multiple_sources_spec.rb
+- functional/consuming_events_from_queue_spec.rb
 - functional/events.dat
 - functional/test_stages.rb
 - lib/empipelines.rb
+- lib/empipelines/aggregated_event_source.rb
 - lib/empipelines/amqp_event_source.rb
 - lib/empipelines/batch_event_source.rb
 - lib/empipelines/event_handlers.rb
@@ -37,6 +40,7 @@ files:
 - lib/empipelines/message.rb
 - lib/empipelines/periodic_event_source.rb
 - lib/empipelines/pipeline.rb
+- unit/empipelines/aggregated_event_source_spec.rb
 - unit/empipelines/amqp_event_source_spec.rb
 - unit/empipelines/batch_event_source_spec.rb
 - unit/empipelines/empty_io_event_source.dat
@@ -64,7 +68,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3827854829014823938
+      hash: -530546752871247334
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: