empipelines 0.2.3 → 0.2.4

Sign up to get free protection for your applications and to get access to all the features.
data/README.md CHANGED
@@ -1,11 +1,22 @@
1
- # EM Pipelines
1
+ # EmPipelines
2
+
3
+ ## Message States
4
+
5
+ * **Consumed:** The message went through all relevant stages and was
6
+ fully consumed.
7
+ * **Broken:** The message *left the EventSource* in an invalid
8
+ state and should not be processed. If the message was corrupted by a
9
+ previous stage it should not be broken, just rejected.
10
+ * **Rejected:** The message is correct but this pipeline instance
11
+ cannot process it, probably due to some temporary problem.
2
12
 
3
13
  ## TODO
4
14
  * Make wiring easier
5
15
  * Example apps
6
- * Control flow for AmqpEventSource
7
16
  * Evented I/O for IOEventSource
8
17
  * Consolidate logger and monitoring
9
18
  * Default monitoring implementation
10
19
  * Performance testing
11
20
  * Detect insonsistency when handler didnt consume or pass message ahead
21
+ * Is callback in the stage's signature redundant? Could we just return
22
+ the message?
data/empipelines.gemspec CHANGED
@@ -4,8 +4,8 @@ Gem::Specification.new do |s|
4
4
  s.rubygems_version = '1.3.5'
5
5
  ## Leave these as is they will be modified for you by the rake gemspec task.
6
6
  s.name = 'empipelines'
7
- s.version = '0.2.3'
8
- s.date = '2012-01-02'
7
+ s.version = '0.2.4'
8
+ s.date = '2012-01-10'
9
9
  s.rubyforge_project = 'empipelines'
10
10
 
11
11
  s.summary = "Simple Event Handling Pipeline Architecture for EventMachine"
@@ -35,10 +35,12 @@ Gem::Specification.new do |s|
35
35
  Gemfile.lock
36
36
  README.md
37
37
  Rakefile
38
+ empipelines-0.2.4.gem
38
39
  empipelines.gemspec
40
+ functional/consuming_events_from_amqp_spec.rb
41
+ functional/consuming_events_from_batch_spec.rb
39
42
  functional/consuming_events_from_file_spec.rb
40
43
  functional/consuming_events_from_multiple_sources_spec.rb
41
- functional/consuming_events_from_queue_spec.rb
42
44
  functional/events.dat
43
45
  functional/test_stages.rb
44
46
  lib/empipelines.rb
@@ -0,0 +1,47 @@
1
+ require 'eventmachine'
2
+ require 'empipelines'
3
+ require 'amqp'
4
+ require 'json'
5
+ require File.join(File.dirname(__FILE__), 'test_stages')
6
+
7
+ module TestStages
8
+ ExchangeName = 'empipelines.build'
9
+ QueueName = 'empipelines.build.queue'
10
+
11
+ describe 'Consumption of events from a in-memory batch' do
12
+ let(:monitoring) { stub(:inform => nil, :debug => nil) }
13
+ let (:processed) { [] }
14
+
15
+ include EmRunner
16
+
17
+ it 'consumes all events from a queue' do
18
+ with_em_run do
19
+ connection = AMQP.connect()
20
+ channel = AMQP::Channel.new(connection)
21
+ channel.prefetch(1)
22
+
23
+ exchange = channel.direct(ExchangeName, :durable => true)
24
+ queue = channel.queue(QueueName, :durable => true)
25
+ queue.bind(exchange)
26
+ queue.purge
27
+
28
+ messages = (1..1000).map { |i| {:data => i}.to_json }
29
+ messages.each { |m| exchange.publish(m, :rounting_key => QueueName) }
30
+
31
+ pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring)
32
+ source = EmPipelines::AmqpEventSource.new(EM, queue, 'msg')
33
+
34
+ stages = [PassthroughStage, PassthroughStage, PassthroughStage]
35
+ event_pipeline = EmPipelines::EventPipeline.new(source, pipeline.for(stages), monitoring)
36
+
37
+ EM.add_periodic_timer(0.1) do
38
+ if processed.size == messages.size * stages.size then
39
+ EM.stop
40
+ end
41
+ end
42
+
43
+ event_pipeline.start!
44
+ end
45
+ end
46
+ end
47
+ end
@@ -3,15 +3,14 @@ require 'empipelines'
3
3
  require File.join(File.dirname(__FILE__), 'test_stages')
4
4
 
5
5
  module TestStages
6
- describe 'Consumption of events from a queue' do
7
- let(:monitoring) { stub() }
8
- let(:logger) { stub(:info => nil, :debug => nil) }
9
- let (:processed) { {} }
6
+ describe 'Consumption of events from a in-memory batch' do
7
+ let(:monitoring) { stub(:inform => nil, :debug => nil) }
8
+ let (:processed) { [] }
10
9
  include EmRunner
11
10
 
12
11
  it 'consumes all events from the a queue' do
13
12
  with_em_run do
14
- pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring, logger)
13
+ pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring)
15
14
 
16
15
  batch = (1...1000).to_a
17
16
  batch_name = "my batch!"
@@ -4,14 +4,13 @@ require File.join(File.dirname(__FILE__), 'test_stages')
4
4
 
5
5
  module TestStages
6
6
  describe 'Consumption of events from a file' do
7
- let(:monitoring) { stub() }
8
- let(:logger) { stub(:info => nil, :debug => nil) }
9
- let (:processed) { {} }
7
+ let(:monitoring) { stub(:inform => nil, :debug => nil) }
8
+ let (:processed) { [] }
10
9
  include EmRunner
11
10
 
12
11
  it 'consumes all events from the file' do
13
12
  with_em_run do
14
- pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring, logger)
13
+ pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring)
15
14
 
16
15
  file_name = File.join(File.dirname(__FILE__), 'events.dat')
17
16
  source = EmPipelines::IOEventSource.new(EM, file_name)
@@ -4,13 +4,12 @@ require File.join(File.dirname(__FILE__), 'test_stages')
4
4
 
5
5
  module TestStages
6
6
  describe 'Consumption of events from multiple sources' do
7
- let(:monitoring) { stub() }
8
- let(:logger) { stub(:info => nil, :debug => nil) }
9
- let (:processed) { {} }
7
+ let(:monitoring) { stub(:inform => nil, :debug => nil) }
8
+ let (:processed) { [] }
10
9
  include EmRunner
11
10
 
12
11
  it 'consumes all events from all sources' do
13
- pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring, logger)
12
+ pipeline = EmPipelines::Pipeline.new(EM, {:processed => processed}, monitoring)
14
13
 
15
14
  file_name = File.join(File.dirname(__FILE__), 'events.dat')
16
15
  num_events_on_file = IO.readlines(file_name).size
@@ -20,7 +20,7 @@ module TestStages
20
20
  end
21
21
 
22
22
  def call(message, &callback)
23
- processed[self.class] = message
23
+ processed << [@id, message.co_id]
24
24
  process(message, callback)
25
25
  end
26
26
  end
@@ -2,7 +2,7 @@ require 'empipelines/event_source'
2
2
 
3
3
  module EmPipelines
4
4
  class AggregatedEventSource < EventSource
5
-
5
+
6
6
  def initialize(em, *event_sources)
7
7
  @em, @sources = em, event_sources.flatten
8
8
  end
@@ -11,13 +11,14 @@ module EmPipelines
11
11
  finished = 0
12
12
  @sources.each do |s|
13
13
  s.on_event(event_handler)
14
-
14
+
15
15
  s.on_finished do |*ignored|
16
16
  finished += 1
17
17
  finished! if finished == @sources.size
18
18
  end
19
+
19
20
  @em.next_tick { s.start! }
20
- end
21
+ end
21
22
  end
22
23
  end
23
24
  end
@@ -12,14 +12,16 @@ module EmPipelines
12
12
  end
13
13
 
14
14
  def start!
15
- @queue.subscribe do |header, json_payload|
15
+ @queue.subscribe(:ack => true) do |header, json_payload|
16
16
  message = Message.new({
17
- :header => header,
18
17
  :origin => @queue.name,
19
18
  :payload => JSON.parse(json_payload),
20
19
  :event => @event_name,
21
20
  :started_at => Time.now.to_i
22
21
  })
22
+ message.on_consumed { |m| header.ack }
23
+ message.on_broken { |m| header.reject(:requeue => true) }
24
+ message.on_rejected { |m| header.reject(:requeue => true) }
23
25
  event!(message)
24
26
  end
25
27
  end
@@ -6,9 +6,8 @@ module EmPipelines
6
6
  end
7
7
  end
8
8
 
9
- def initialize(em, context, monitoring, logger)
9
+ def initialize(em, context, monitoring)
10
10
  @em = em
11
- @logger = logger
12
11
  @context = context
13
12
  @monitoring = monitoring
14
13
  end
@@ -17,12 +16,11 @@ module EmPipelines
17
16
  stages = event_definition.map(&instantiate_with_dependencies)
18
17
 
19
18
  monitoring = @monitoring
20
- logger = @logger
21
19
 
22
20
  first_stage_process = stages.reverse.reduce(TerminatorStage) do |current_head, next_stage|
23
21
  @em.spawn do |input|
24
22
  begin
25
- logger.debug "#{next_stage.class}#notify with #{input}}"
23
+ monitoring.debug "#{next_stage.class}#notify with #{input}}"
26
24
  next_stage.call(input) do |output|
27
25
  current_head.notify(output)
28
26
  end
@@ -32,7 +30,7 @@ module EmPipelines
32
30
  end
33
31
  end
34
32
 
35
- @logger.info "Pipeline for event_definition is: #{stages.map(&:class).join('->')}"
33
+ @monitoring.inform "Pipeline for event_definition is: #{stages.map(&:class).join('->')}"
36
34
  first_stage_process
37
35
  end
38
36
 
data/lib/empipelines.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  module EmPipelines
2
- VERSION = '0.2.3'
2
+ VERSION = '0.2.4'
3
3
  end
4
4
 
5
5
  require 'empipelines/message'
@@ -3,7 +3,7 @@ require 'empipelines/amqp_event_source'
3
3
  module EmPipelines
4
4
  class StubQueue
5
5
  attr_accessor :name
6
- def subscribe(&code)
6
+ def subscribe(ack, &code)
7
7
  @code = code
8
8
  end
9
9
 
@@ -11,7 +11,7 @@ module EmPipelines
11
11
  @code.call(header, payload)
12
12
  end
13
13
  end
14
-
14
+
15
15
  describe AmqpEventSource do
16
16
  let (:em) { mock('eventmachine') }
17
17
 
@@ -23,7 +23,7 @@ module EmPipelines
23
23
  header = stub('header')
24
24
 
25
25
  received_messages = []
26
-
26
+
27
27
  amqp_source = AmqpEventSource.new(em, queue, event_type)
28
28
  amqp_source.on_event { |e| received_messages << e }
29
29
  amqp_source.start!
@@ -32,10 +32,48 @@ module EmPipelines
32
32
 
33
33
  received_messages.size.should eql(1)
34
34
  received_messages.first[:origin].should ==(queue.name)
35
- received_messages.first[:header].should ==(header)
36
35
  received_messages.first[:payload].should ==({:key => "value"})
37
36
  received_messages.first[:event].should ==(event_type)
38
37
  received_messages.first[:started_at].should_not be_nil
39
38
  end
39
+
40
+ it 'acknowledges consumed messages' do
41
+ queue = StubQueue.new
42
+ header = mock('header')
43
+
44
+ header.should_receive(:ack)
45
+
46
+ amqp_source = AmqpEventSource.new(em, queue, 'event type')
47
+ amqp_source.on_event { |e| e.consumed! }
48
+ amqp_source.start!
49
+
50
+ queue.publish(header, '{"key":"value"}')
51
+ end
52
+
53
+ it 'rejects broken messages with no requeue' do
54
+ queue = StubQueue.new
55
+ header = mock('header')
56
+
57
+ header.should_receive(:reject)
58
+
59
+ amqp_source = AmqpEventSource.new(em, queue, 'event type')
60
+ amqp_source.on_event { |e| e.broken! }
61
+ amqp_source.start!
62
+
63
+ queue.publish(header, '{"key":"value"}')
64
+ end
65
+
66
+ it 'rejects rejected messages with requeue' do
67
+ queue = StubQueue.new
68
+ header = mock('header')
69
+
70
+ header.should_receive(:reject).with({:requeue => true})
71
+
72
+ amqp_source = AmqpEventSource.new(em, queue, 'event type')
73
+ amqp_source.on_event { |e| e.rejected! }
74
+ amqp_source.start!
75
+
76
+ queue.publish(header, '{"key":"value"}')
77
+ end
40
78
  end
41
79
  end
@@ -96,14 +96,14 @@ module EmPipelines
96
96
  end
97
97
 
98
98
  describe Pipeline do
99
- let(:logger) {stub(:info => true, :debug => true)}
99
+ let(:monitoring) { stub(:inform => nil, :debug => nil) }
100
100
 
101
101
  it 'chains the actions using processes' do
102
102
  event_chain = [AddOne, SquareIt, GlobalHolder]
103
103
  a_msg = msg({:data =>1})
104
104
  a_msg.should_receive(:consumed!)
105
105
 
106
- pipelines = Pipeline.new(StubSpawner.new, {}, stub('monitoring'), logger)
106
+ pipelines = Pipeline.new(StubSpawner.new, {}, monitoring)
107
107
  pipeline = pipelines.for(event_chain)
108
108
  pipeline.notify(a_msg)
109
109
 
@@ -112,7 +112,7 @@ module EmPipelines
112
112
 
113
113
  it 'does not send to the next if last returned nil' do
114
114
  event_chain = [AddOne, SquareIt, DeadEnd, GlobalHolder]
115
- pipelines = Pipeline.new(StubSpawner.new, {}, stub('monitoring'), logger)
115
+ pipelines = Pipeline.new(StubSpawner.new, {}, monitoring)
116
116
  pipeline = pipelines.for(event_chain)
117
117
  pipeline.notify(msg({:data => 1}))
118
118
  GlobalHolder.held.should be_nil
@@ -120,7 +120,7 @@ module EmPipelines
120
120
 
121
121
  it 'makes all objects in the context object available to stages' do
122
122
  event_chain = [NeedsAnApple, NeedsAnOrange, GlobalHolder]
123
- pipelines = Pipeline.new(StubSpawner.new, {:apple => :some_apple, :orange => :some_orange}, stub('monitoring'), logger)
123
+ pipelines = Pipeline.new(StubSpawner.new, {:apple => :some_apple, :orange => :some_orange}, monitoring)
124
124
  a_msg = msg({})
125
125
  a_msg.should_receive(:consumed!)
126
126
 
@@ -132,15 +132,14 @@ module EmPipelines
132
132
  end
133
133
 
134
134
  it 'sends exception to the proper handler' do
135
- monitoring = mock()
136
135
  monitoring.should_receive(:inform_exception!)
137
- pipeline = Pipeline.new(StubSpawner.new, {}, monitoring, logger)
136
+ pipeline = Pipeline.new(StubSpawner.new, {}, monitoring)
138
137
  pipeline.for([BrokenStage]).notify(msg({}))
139
138
  end
140
139
 
141
140
  it 'flags the message as consumed if goest through all stages' do
142
141
  event_chain = [Passthrough, Passthrough]
143
- pipelines = Pipeline.new(StubSpawner.new, {}, stub('monitoring'), logger)
142
+ pipelines = Pipeline.new(StubSpawner.new, {}, monitoring)
144
143
  pipeline = pipelines.for(event_chain)
145
144
  a_msg = msg({:data => :whatevah})
146
145
  a_msg.should_receive(:consumed!)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: empipelines
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.3
4
+ version: 0.2.4
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2012-01-02 00:00:00.000000000Z
13
+ date: 2012-01-10 00:00:00.000000000Z
14
14
  dependencies: []
15
15
  description: Simple Event Handling Pipeline Architecture for EventMachine
16
16
  email: pcalcado+empipelines@gmail.com
@@ -23,10 +23,12 @@ files:
23
23
  - Gemfile.lock
24
24
  - README.md
25
25
  - Rakefile
26
+ - empipelines-0.2.4.gem
26
27
  - empipelines.gemspec
28
+ - functional/consuming_events_from_amqp_spec.rb
29
+ - functional/consuming_events_from_batch_spec.rb
27
30
  - functional/consuming_events_from_file_spec.rb
28
31
  - functional/consuming_events_from_multiple_sources_spec.rb
29
- - functional/consuming_events_from_queue_spec.rb
30
32
  - functional/events.dat
31
33
  - functional/test_stages.rb
32
34
  - lib/empipelines.rb
@@ -67,7 +69,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
67
69
  version: '0'
68
70
  segments:
69
71
  - 0
70
- hash: 2169333879126328078
72
+ hash: 4556679468386328917
71
73
  required_rubygems_version: !ruby/object:Gem::Requirement
72
74
  none: false
73
75
  requirements: