empipelines 0.2 → 0.2.1

data/README.md

@@ -1 +1,14 @@
 # EM Pipelines
+
+## TODO
+* Make wiring easier
+* Example apps
+* TerminatorStage should consume message
+* Control flow for AmqpEventSource
+* Transaction ID on each message
+* Evented I/O for IOEventSource
+* Consolidate logger and monitoring
+* Default monitoring implementation
+* Performance testing
+* Make all events composable (e.g. new_on_event = old_on_event o new_handler)
+* Detect insonsistency when handler didnt consume or pass message ahead

data/Rakefile

@@ -59,12 +59,23 @@ Cucumber::Rake::Task.new(:all_features) do |t|
   t.cucumber_opts = "--format pretty"
 end
 
-desc "Runs specs"
-task :specs do
-  all = FileList['spec/**/*_spec.rb']
+def spec(dir)
+  puts "---------- #{dir} tests... ----------"
+  all = FileList["#{dir}/**/*_spec.rb"]
   sh "rspec --color  #{all}"
 end
 
+desc "Runs unit tests"
+task :unit do
+  spec('unit')
+end
+
+desc "Run functional tests"
+task :functional do
+  spec('functional')
+end
+
+
 desc "Resets localhost's rabbitmq"
 task :reset_rabbitmq do
 #  sh 'rabbitmqctl stop_app; echo 0'
@@ -72,17 +83,7 @@ task :reset_rabbitmq do
 #  sh 'rabbitmqctl start_app'
 end
 
-desc "Run cucumber tests for finished features"
-task :features do
-
-end
-
-desc "Run cucumber tests for all features, including work in progress"
-task :all_features do
-  
-end
-
-task :ci => [:specs, :features, :build]
+task :ci => [:unit, :functional, :build]
 
 desc "MUST BE RUN (AND PASS!) BEFORE CHECKING IN CODE!"
 task :pre_checkin => [:reset_rabbitmq, :ci]

data/empipelines.gemspec

@@ -4,8 +4,8 @@ Gem::Specification.new do |s|
   s.rubygems_version = '1.3.5'
   ## Leave these as is they will be modified for you by the rake gemspec task.
   s.name              = 'empipelines'
-  s.version           = '0.2'
-  s.date              = '2011-12-20'
+  s.version           = '0.2.1'
+  s.date              = '2011-12-21'
   s.rubyforge_project = 'empipelines'
 
   s.summary           = "Simple Event Handling Pipeline Architecture for EventMachine"
@@ -36,23 +36,31 @@ Gem::Specification.new do |s|
     README.md
     Rakefile
     empipelines.gemspec
+    functional/consuming_events_from_file_spec.rb
+    functional/events.dat
+    functional/test_stages.rb
     lib/empipelines.rb
     lib/empipelines/amqp_event_source.rb
     lib/empipelines/batch_event_source.rb
+    lib/empipelines/event_handlers.rb
     lib/empipelines/event_pipeline.rb
+    lib/empipelines/io_event_source.rb
     lib/empipelines/list_event_source.rb
     lib/empipelines/message.rb
     lib/empipelines/periodic_event_source.rb
     lib/empipelines/pipeline.rb
-    spec/empipelines/amqp_event_source_spec.rb
-    spec/empipelines/batch_event_source_spec.rb
-    spec/empipelines/event_pipeline_spec.rb
-    spec/empipelines/list_event_source_spec.rb
-    spec/empipelines/message_spec.rb
-    spec/empipelines/periodic_event_source_spec.rb
-    spec/empipelines/pipeline_spec.rb
-    spec/spec_helper.rb
-    spec/stage_helper.rb
+    unit/empipelines/amqp_event_source_spec.rb
+    unit/empipelines/batch_event_source_spec.rb
+    unit/empipelines/empty_io_event_source.dat
+    unit/empipelines/event_pipeline_spec.rb
+    unit/empipelines/io_event_source.dat
+    unit/empipelines/io_event_source_spec.rb
+    unit/empipelines/list_event_source_spec.rb
+    unit/empipelines/message_spec.rb
+    unit/empipelines/periodic_event_source_spec.rb
+    unit/empipelines/pipeline_spec.rb
+    unit/spec_helper.rb
+    unit/stage_helper.rb
   ]
   # = MANIFEST =
 

data/functional/consuming_events_from_file_spec.rb

@@ -0,0 +1,42 @@
+require 'eventmachine'
+require 'empipelines'
+require File.join(File.dirname(__FILE__), 'test_stages')
+
+module TestStages
+  describe 'Consumption of events from a file' do
+    let(:monitoring) { stub() }
+    let(:logger) { stub(:info => nil, :debug => nil) }
+    let (:processed) { {} }
+    include EmRunner
+
+    it 'consumes all events from the file' do
+      with_em_run do      
+        pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring, logger)
+
+        file_name = File.join(File.dirname(__FILE__), 'events.dat')
+        source = EmPipelines::IOEventSource.new(EM, file_name)
+
+        stages = [PassthroughStage, PassthroughStage, PassthroughStage, ConsumeStage]
+        event_pipeline = EmPipelines::EventPipeline.new(source, pipeline.for(stages), monitoring)
+
+        source.on_finished do |messages|
+          EM.stop  
+          messages.should have(10).items
+          messages[0][:payload].should ==("event #0")
+          messages[1][:payload].should ==("event #1")
+          messages[2][:payload].should ==("event #2")
+          messages[3][:payload].should ==("event #3")
+          messages[4][:payload].should ==("event #4")
+          messages[5][:payload].should ==("event #5")
+          messages[6][:payload].should ==("event #6")
+          messages[7][:payload].should ==("event #7")
+          messages[8][:payload].should ==("event #8")
+          messages[9][:payload].should ==("event #9")
+          messages.each { |m| m.state.should ==(:consumed) }        
+        end
+        
+        event_pipeline.start!
+      end
+    end
+  end
+end

data/functional/events.dat

@@ -0,0 +1,10 @@
+event #0
+event #1
+event #2
+event #3
+event #4
+event #5
+event #6
+event #7
+event #8
+event #9

data/functional/test_stages.rb

@@ -0,0 +1,67 @@
+module TestStages
+  module EmRunner
+    def with_em_run(&test_body)
+      EM.run do
+        EM.add_timer(10) do
+          EM.stop
+          raise "################### Test timed-out! ################### "
+        end
+
+        test_body.call
+      end
+    end
+  end
+
+  module SomeStage
+    @@last_id = 0
+    def initialize
+      @id = "module ##{(@@last_id)}"
+      @@last_id += 1
+    end
+    
+    def call(message, &callback)
+      processed[self.class] = message
+      process(message, callback)
+    end          
+  end
+  
+  class PassthroughStage
+    include SomeStage
+    
+    def process(message, callback)
+      callback.call(message)
+    end
+  end
+
+  class ShouldNotBeReachedStage
+    include SomeStage
+    
+    def process(message, callback)
+      raise "should not be reached but got #{message}!"
+    end
+  end
+
+  class ConsumeStage
+    include SomeStage
+    
+    def process(message, callback)
+      message.consumed!
+    end
+  end
+  
+  class RejectStage
+    include SomeStage
+    
+    def process(message, callback)
+      message.rejected!
+    end
+  end
+  
+  class BrokenMessageStage
+    include SomeStage
+    
+    def process(message, callback)
+      message.broken!
+    end
+  end
+end

data/lib/empipelines/amqp_event_source.rb

@@ -2,8 +2,8 @@ require 'json'
 
 module EmPipelines
   class AmqpEventSource
-    def initialize(queue, event_name)
-      @queue, @event_name = queue, event_name
+    def initialize(em, queue, event_name)
+      @em, @queue, @event_name = em, queue, event_name
     end
 
     def on_event(&handler)
@@ -13,10 +13,11 @@ module EmPipelines
     def start!
       @queue.subscribe do |header, json_payload|
         message = Message.new ({
-                                :header     => header,
-                                :payload    => JSON.parse(json_payload),
-                                :event      => @event_name,
-                                :started_at => Time.now.to_i
+                                 :header     => header,
+                                 :origin     => @queue.name,
+                                 :payload    => JSON.parse(json_payload),
+                                 :event      => @event_name,
+                                 :started_at => Time.now.to_i
                                 })
         @handler.call(message)
       end

data/lib/empipelines/batch_event_source.rb

@@ -1,15 +1,11 @@
+require 'empipelines/event_handlers'
+
 module EmPipelines
   class BatchEventSource
-    def initialize(em, events)
-      @em, @events = em, events
-    end
-
-    def on_event(&handler)
-      @handler = handler
-    end
-
-    def on_batch_finished(&batch_finished_handler)
-      @batch_finished_handler = batch_finished_handler
+    include EventHandlers
+    
+    def initialize(em, list_name, events)
+      @em, @list_name, @events = em, list_name,events
     end
 
     def start!
@@ -22,13 +18,16 @@ module EmPipelines
       end
 
       @events.each do |e|
-        message = Message.new({:payload => e})
+        message = Message.new({
+                                :payload => e,
+                                :origin => @list_name
+                              })
 
         message.on_rejected_broken(message_finished)
         message.on_rejected(message_finished)
         message.on_consumed(message_finished)
 
-        @handler.call(message)
+        event_handler.call(message)
       end
     end
 
@@ -36,8 +35,8 @@ module EmPipelines
     def check_if_finished
       finished = (@finalised.size == @events.size)
 
-      if finished and @batch_finished_handler
-        @em.next_tick { @batch_finished_handler.call(@finalised) }
+      if finished and finished_handler
+        @em.next_tick { finished_handler.call(@finalised) }
       end
     end
   end

data/lib/empipelines/event_handlers.rb

@@ -0,0 +1,20 @@
+module EmPipelines
+  module EventHandlers
+    def on_event(event_handler=nil, &block)
+      @event_handler = block_given? ? block : event_handler
+    end
+
+    def on_finished(batch_finished_handler=nil, &block)
+      @finished_handler = block_given? ? block : batch_finished_handler
+    end
+
+    protected
+    def finished_handler
+      @finished_handler
+    end
+
+    def event_handler
+      @event_handler
+    end
+  end
+end    

data/lib/empipelines/io_event_source.rb

@@ -0,0 +1,22 @@
+require 'empipelines/event_handlers'
+
+module EmPipelines
+  class IOEventSource
+    include EventHandlers
+
+    def initialize(em, file_path)
+      raise "File #{file_path} does not exist!" unless File.exists?(file_path)
+      @em, @file_path = em, file_path
+    end
+    
+    def start!
+      #TODO: this sucks hard, move to evented I/O
+      events = IO.readlines(@file_path).map { |e| e.strip }
+
+      wrapped_handler = BatchEventSource.new(@em, @file_path, events)
+      wrapped_handler.on_event(event_handler)
+      wrapped_handler.on_finished(finished_handler)
+      wrapped_handler.start!
+    end
+  end
+end

data/lib/empipelines/message.rb

@@ -1,5 +1,7 @@
 module EmPipelines
   class Message
+    attr_reader :state
+    
     def initialize(base_hash={})
       hash!(base_hash)
       created!

data/lib/empipelines/periodic_event_source.rb

@@ -1,7 +1,10 @@
 module EmPipelines
   class PeriodicEventSource
-    def initialize(em, interval_in_secs, &event_sourcing_code)
-      @em, @interval_in_secs, @event_sourcing_code = em, interval_in_secs, event_sourcing_code
+    def initialize(em, name, interval_in_secs, &event_sourcing_code)
+      @em                  = em
+      @name                = name
+      @interval_in_secs    = interval_in_secs
+      @event_sourcing_code = event_sourcing_code 
     end
 
     def start!
@@ -18,7 +21,8 @@ module EmPipelines
 
     def tick!      
       event = @event_sourcing_code.call
-      @handler.call(event) if event
+      
+      @handler.call(Message.new(:payload => event, :origin => @name)) if event
     end
   end
 end

data/lib/empipelines/pipeline.rb

@@ -6,8 +6,8 @@ module EmPipelines
       end
     end
     
-    def initialize(spawner, context, monitoring, logger)
-      @spawner = spawner
+    def initialize(em, context, monitoring, logger)
+      @em = em
       @logger = logger
       @context = context
       @monitoring = monitoring
@@ -20,7 +20,7 @@ module EmPipelines
       logger = @logger
       
       first_stage_process = stages.reverse.reduce(TerminatorStage) do |current_head, next_stage|
-        @spawner.spawn do |input|
+        @em.spawn do |input|
           begin
             logger.debug "#{next_stage.class}#notify with #{input}}"
             next_stage.call(input) do |output|

data/lib/empipelines.rb

@@ -1,9 +1,11 @@
 module EmPipelines
-  VERSION = '0.2'
+  VERSION = '0.2.1'
 end
 
+require 'empipelines/event_handlers'
 require 'empipelines/amqp_event_source'
 require 'empipelines/batch_event_source'
+require 'empipelines/io_event_source'
 require 'empipelines/event_pipeline'
 require 'empipelines/list_event_source'
 require 'empipelines/message'

data/{spec → unit}/empipelines/amqp_event_source_spec.rb

@@ -2,6 +2,7 @@ require 'empipelines/amqp_event_source'
 
 module EmPipelines
   class StubQueue
+    attr_accessor :name
     def subscribe(&code)
       @code = code
     end
@@ -12,21 +13,25 @@ module EmPipelines
   end
   
   describe AmqpEventSource do
-    it "adds payload, time and event type to message and sends to listener" do
+    let (:em) { mock('eventmachine') }
+
+    it 'wraps each AMQP message and send to listeners' do
       json_payload = '{"key":"value"}'
       queue = StubQueue.new
+      queue.name = "this.is.some.queue"
       event_type = "NuclearWar"
       header = stub('header')
 
       received_messages = []
       
-      amqp_source = AmqpEventSource.new(queue, event_type)
+      amqp_source = AmqpEventSource.new(em, queue, event_type)
       amqp_source.on_event { |e| received_messages << e }
       amqp_source.start!
 
       queue.publish(header, json_payload)
 
       received_messages.size.should eql(1)
+      received_messages.first[:origin].should ==(queue.name)
       received_messages.first[:header].should ==(header)
       received_messages.first[:payload].should ==({:key => "value"})
       received_messages.first[:event].should ==(event_type)

data/{spec → unit}/empipelines/batch_event_source_spec.rb

@@ -9,9 +9,11 @@ module EmPipelines
       em
     end
 
+    let (:list_name) { "list of stuff"  }
+
     it "sends each element on the list as a payload to the listener" do
       events = [1,2,3,4,5,6,7,8,9,10]
-      source = BatchEventSource.new(em, events)
+      source = BatchEventSource.new(em, list_name, events)
 
       received = []
       source.on_event do |e|
@@ -21,15 +23,16 @@ module EmPipelines
       source.start!
 
       received.map{ |i| i[:payload] }.should ==(events)
+      received.each{ |i| i[:origin].should == list_name }
     end
 
     it "calls the batch finished callback when all items were processed" do
       events = [1,2,3,4,5,6,7,8,9,10]
-      source = BatchEventSource.new(em, events)
+      source = BatchEventSource.new(em, list_name, events)
 
       has_finished = []
 
-      source.on_batch_finished do |messages|
+      source.on_finished do |messages|
         has_finished << messages
       end
 
@@ -42,11 +45,11 @@ module EmPipelines
       has_finished.first.map{ |i| i[:payload] }.should ==(events)
     end
 
-    it "finishes straight away if there are no events to process" do
-      source = BatchEventSource.new(em, [])
+    it "finishes immediately if there are no events to process" do
+      source = BatchEventSource.new(em, list_name, [])
 
       has_finished = []
-      source.on_batch_finished do |messages|
+      source.on_finished do |messages|
         has_finished << true
       end
 
@@ -59,11 +62,11 @@ module EmPipelines
       has_finished.first.should be_true
     end
 
-    it "does not call the batch finished callback if not all of the  items were processed" do
+    it "only calls the finished handler if all events were processed" do
       events = [1,2,3,4,5,6,7,8,9,10]
-      source = BatchEventSource.new(em, events)
+      source = BatchEventSource.new(em, list_name, events)
 
-      source.on_batch_finished do |messages|
+      source.on_finished do |messages|
         raise "should not be called"
       end
 

data/unit/empipelines/io_event_source.dat

@@ -0,0 +1,3 @@
+1
+2
+3

data/unit/empipelines/io_event_source_spec.rb

@@ -0,0 +1,87 @@
+require 'empipelines/io_event_source'
+
+module EmPipelines
+  describe IOEventSource do    
+    let(:events_file) { File.join(File.dirname(__FILE__), 'io_event_source.dat')}
+    let(:empty_file) { File.join(File.dirname(__FILE__), 'empty_io_event_source.dat')}
+    let(:inexistent_file) { File.join(File.dirname(__FILE__), 'not_really_son.dat')}
+    let (:em) do
+      em = mock('eventmachine')
+      em.stub(:next_tick).and_yield
+      em
+    end
+
+    it 'verifies file existance'  do
+      lambda{ IOEventSource.new(em, inexistent_file) }.should raise_error
+    end
+    
+    it 'sends each line in the file as a message to listeners' do
+      source = IOEventSource.new(em, events_file)
+
+      received = []
+      source.on_event do |e|
+        received << e
+      end
+
+      source.start!
+
+      received.map{ |i| i[:payload].to_i }.should ==([1,2,3])
+      received.each{ |i| i[:origin].should == events_file }
+    end
+
+    
+    it 'calls the finished callback when all messages were processed' do
+      source = IOEventSource.new(em, events_file)
+
+      has_finished = []
+
+      source.on_finished do |messages|
+        has_finished << messages
+      end
+
+      source.on_event do |e|
+        e.consumed!
+      end
+
+      source.start!
+
+      has_finished.first.map{ |i| i[:payload].to_i }.should ==([1,2,3])
+    end
+    
+    it 'finishes immediately if there are no events to process' do
+      source = IOEventSource.new(em, empty_file)
+
+      has_finished = []
+      source.on_finished do |messages|
+        has_finished << true
+      end
+
+      source.on_event do |e|
+        raise 'should not be called!'
+      end
+
+      source.start!
+
+      has_finished.first.should be_true
+    end
+    
+    it 'only calls the finished handler if all events were processed' do
+      source = IOEventSource.new(em, events_file)
+
+      source.on_finished do |messages|
+        raise "should not be called"
+      end
+
+      count = 0
+      source.on_event do |e|
+        e.consumed! if (count=+1) > 1
+      end
+
+      source.start!
+    end
+
+    it 'can read really long files' do
+      pending('current implementation is terribly naive')
+    end
+  end
+end

data/unit/empipelines/periodic_event_source_spec.rb

@@ -0,0 +1,31 @@
+require 'empipelines/periodic_event_source'
+
+module EmPipelines
+  describe PeriodicEventSource do
+    it 'schedules itself with eventmachine' do
+      em = stub('eventmachine')
+      em.should_receive(:add_periodic_timer).with(5)
+      PeriodicEventSource.new(em, 'a name', 5){ 'something cool!' }.start!
+    end
+    
+    it 'sends the result of the periodic action to the handler' do
+      name = 'some name'
+      event = {:this => :is, :some => 'event'}
+      received_messages = []      
+
+      source = PeriodicEventSource.new(stub('eventmachine'), name, 666){ event }
+      source.on_event { |msg| received_messages << msg}      
+      source.tick!
+
+      received_messages.should have(1).item
+      received_messages[0][:payload].should eql(event)
+      received_messages[0][:origin].should eql(name)
+    end
+     
+    it 'doesnt o anything if no event was generated' do
+      source = PeriodicEventSource.new(stub('eventmachine'), 'a name', 666){ nil }
+      source.on_event { |m| raise 'should not be called' }
+      source.tick!
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: empipelines
 version: !ruby/object:Gem::Version
-  version: '0.2'
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-12-20 00:00:00.000000000Z
+date: 2011-12-21 00:00:00.000000000Z
 dependencies: []
 description: Simple Event Handling Pipeline Architecture for EventMachine
 email: pcalcado+empipelines@gmail.com
@@ -24,23 +24,31 @@ files:
 - README.md
 - Rakefile
 - empipelines.gemspec
+- functional/consuming_events_from_file_spec.rb
+- functional/events.dat
+- functional/test_stages.rb
 - lib/empipelines.rb
 - lib/empipelines/amqp_event_source.rb
 - lib/empipelines/batch_event_source.rb
+- lib/empipelines/event_handlers.rb
 - lib/empipelines/event_pipeline.rb
+- lib/empipelines/io_event_source.rb
 - lib/empipelines/list_event_source.rb
 - lib/empipelines/message.rb
 - lib/empipelines/periodic_event_source.rb
 - lib/empipelines/pipeline.rb
-- spec/empipelines/amqp_event_source_spec.rb
-- spec/empipelines/batch_event_source_spec.rb
-- spec/empipelines/event_pipeline_spec.rb
-- spec/empipelines/list_event_source_spec.rb
-- spec/empipelines/message_spec.rb
-- spec/empipelines/periodic_event_source_spec.rb
-- spec/empipelines/pipeline_spec.rb
-- spec/spec_helper.rb
-- spec/stage_helper.rb
+- unit/empipelines/amqp_event_source_spec.rb
+- unit/empipelines/batch_event_source_spec.rb
+- unit/empipelines/empty_io_event_source.dat
+- unit/empipelines/event_pipeline_spec.rb
+- unit/empipelines/io_event_source.dat
+- unit/empipelines/io_event_source_spec.rb
+- unit/empipelines/list_event_source_spec.rb
+- unit/empipelines/message_spec.rb
+- unit/empipelines/periodic_event_source_spec.rb
+- unit/empipelines/pipeline_spec.rb
+- unit/spec_helper.rb
+- unit/stage_helper.rb
 homepage: http://github.com/pcalcado/empipelines
 licenses: []
 post_install_message: 
@@ -56,7 +64,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4504897307794021270
+      hash: -3827854829014823938
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:

data/spec/empipelines/periodic_event_source_spec.rb

@@ -1,28 +0,0 @@
-require 'empipelines/periodic_event_source'
-
-module EmPipelines
-  describe PeriodicEventSource do
-    it 'schedules itself with eventmachine' do
-      em = stub('eventmachine')
-      em.should_receive(:add_periodic_timer).with(5)
-      PeriodicEventSource.new(em, 5){ "something cool!" }.start!
-    end
-    
-    it 'sends the result of the periodic action to the handler' do
-      expected_message = { :something => "goes here" }
-      received_messages = []
-
-      source = PeriodicEventSource.new(stub('eventmachine'), 666){ expected_message }
-      source.on_event { |msg| received_messages << msg}      
-      source.tick!
-
-      received_messages.should eql([expected_message])      
-    end
-    
-    it 'doesnt o anything if no event was generated' do      
-      source = PeriodicEventSource.new(stub('eventmachine'), 666){ nil }
-      source.on_event { |m| raise "should not be called" }      
-      source.tick!
-    end
-  end  
-end