empipelines 0.2.2 → 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +0 -2
- data/empipelines.gemspec +4 -5
- data/lib/empipelines/aggregated_event_source.rb +3 -4
- data/lib/empipelines/amqp_event_source.rb +13 -13
- data/lib/empipelines/batch_event_source.rb +6 -10
- data/lib/empipelines/event_source.rb +67 -0
- data/lib/empipelines/io_event_source.rb +3 -4
- data/lib/empipelines/message.rb +39 -18
- data/lib/empipelines/periodic_event_source.rb +2 -6
- data/lib/empipelines.rb +2 -3
- data/unit/empipelines/aggregated_event_source_spec.rb +15 -17
- data/unit/empipelines/batch_event_source_spec.rb +2 -4
- data/unit/empipelines/event_source_spec.rb +115 -0
- data/unit/empipelines/io_event_source_spec.rb +1 -0
- data/unit/empipelines/message_spec.rb +160 -41
- data/unit/empipelines/pipeline_spec.rb +1 -0
- metadata +5 -6
- data/lib/empipelines/event_handlers.rb +0 -20
- data/lib/empipelines/list_event_source.rb +0 -17
- data/unit/empipelines/list_event_source_spec.rb +0 -17
data/README.md
CHANGED
@@ -4,10 +4,8 @@
|
|
4
4
|
* Make wiring easier
|
5
5
|
* Example apps
|
6
6
|
* Control flow for AmqpEventSource
|
7
|
-
* Transaction ID on each message
|
8
7
|
* Evented I/O for IOEventSource
|
9
8
|
* Consolidate logger and monitoring
|
10
9
|
* Default monitoring implementation
|
11
10
|
* Performance testing
|
12
|
-
* Make all events composable (e.g. new_on_event = old_on_event o new_handler)
|
13
11
|
* Detect insonsistency when handler didnt consume or pass message ahead
|
data/empipelines.gemspec
CHANGED
@@ -4,8 +4,8 @@ Gem::Specification.new do |s|
|
|
4
4
|
s.rubygems_version = '1.3.5'
|
5
5
|
## Leave these as is they will be modified for you by the rake gemspec task.
|
6
6
|
s.name = 'empipelines'
|
7
|
-
s.version = '0.2.
|
8
|
-
s.date = '
|
7
|
+
s.version = '0.2.3'
|
8
|
+
s.date = '2012-01-02'
|
9
9
|
s.rubyforge_project = 'empipelines'
|
10
10
|
|
11
11
|
s.summary = "Simple Event Handling Pipeline Architecture for EventMachine"
|
@@ -45,10 +45,9 @@ Gem::Specification.new do |s|
|
|
45
45
|
lib/empipelines/aggregated_event_source.rb
|
46
46
|
lib/empipelines/amqp_event_source.rb
|
47
47
|
lib/empipelines/batch_event_source.rb
|
48
|
-
lib/empipelines/event_handlers.rb
|
49
48
|
lib/empipelines/event_pipeline.rb
|
49
|
+
lib/empipelines/event_source.rb
|
50
50
|
lib/empipelines/io_event_source.rb
|
51
|
-
lib/empipelines/list_event_source.rb
|
52
51
|
lib/empipelines/message.rb
|
53
52
|
lib/empipelines/periodic_event_source.rb
|
54
53
|
lib/empipelines/pipeline.rb
|
@@ -57,9 +56,9 @@ Gem::Specification.new do |s|
|
|
57
56
|
unit/empipelines/batch_event_source_spec.rb
|
58
57
|
unit/empipelines/empty_io_event_source.dat
|
59
58
|
unit/empipelines/event_pipeline_spec.rb
|
59
|
+
unit/empipelines/event_source_spec.rb
|
60
60
|
unit/empipelines/io_event_source.dat
|
61
61
|
unit/empipelines/io_event_source_spec.rb
|
62
|
-
unit/empipelines/list_event_source_spec.rb
|
63
62
|
unit/empipelines/message_spec.rb
|
64
63
|
unit/empipelines/periodic_event_source_spec.rb
|
65
64
|
unit/empipelines/pipeline_spec.rb
|
@@ -1,8 +1,7 @@
|
|
1
|
-
require 'empipelines/
|
1
|
+
require 'empipelines/event_source'
|
2
2
|
|
3
3
|
module EmPipelines
|
4
|
-
class AggregatedEventSource
|
5
|
-
include EventHandlers
|
4
|
+
class AggregatedEventSource < EventSource
|
6
5
|
|
7
6
|
def initialize(em, *event_sources)
|
8
7
|
@em, @sources = em, event_sources.flatten
|
@@ -15,7 +14,7 @@ module EmPipelines
|
|
15
14
|
|
16
15
|
s.on_finished do |*ignored|
|
17
16
|
finished += 1
|
18
|
-
|
17
|
+
finished! if finished == @sources.size
|
19
18
|
end
|
20
19
|
@em.next_tick { s.start! }
|
21
20
|
end
|
@@ -1,26 +1,26 @@
|
|
1
|
+
require 'empipelines/event_source'
|
2
|
+
require 'empipelines/message'
|
3
|
+
|
1
4
|
require 'json'
|
2
5
|
|
3
6
|
module EmPipelines
|
4
7
|
#this must have a on_finished!
|
5
|
-
class AmqpEventSource
|
8
|
+
class AmqpEventSource < EventSource
|
9
|
+
|
6
10
|
def initialize(em, queue, event_name)
|
7
11
|
@em, @queue, @event_name = em, queue, event_name
|
8
12
|
end
|
9
13
|
|
10
|
-
def on_event(&handler)
|
11
|
-
@handler = handler
|
12
|
-
end
|
13
|
-
|
14
14
|
def start!
|
15
15
|
@queue.subscribe do |header, json_payload|
|
16
|
-
message = Message.new
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
16
|
+
message = Message.new({
|
17
|
+
:header => header,
|
18
|
+
:origin => @queue.name,
|
19
|
+
:payload => JSON.parse(json_payload),
|
20
|
+
:event => @event_name,
|
21
|
+
:started_at => Time.now.to_i
|
22
|
+
})
|
23
|
+
event!(message)
|
24
24
|
end
|
25
25
|
end
|
26
26
|
end
|
@@ -1,8 +1,8 @@
|
|
1
|
-
require 'empipelines/
|
1
|
+
require 'empipelines/message'
|
2
|
+
require 'empipelines/event_source'
|
2
3
|
|
3
4
|
module EmPipelines
|
4
|
-
class BatchEventSource
|
5
|
-
include EventHandlers
|
5
|
+
class BatchEventSource < EventSource
|
6
6
|
|
7
7
|
def initialize(em, list_name, events)
|
8
8
|
@num_finalised = 0
|
@@ -24,11 +24,11 @@ module EmPipelines
|
|
24
24
|
:origin => @list_name
|
25
25
|
})
|
26
26
|
|
27
|
-
message.
|
27
|
+
message.on_broken(message_finished)
|
28
28
|
message.on_rejected(message_finished)
|
29
29
|
message.on_consumed(message_finished)
|
30
30
|
|
31
|
-
|
31
|
+
event!(message)
|
32
32
|
end
|
33
33
|
end
|
34
34
|
|
@@ -37,11 +37,7 @@ module EmPipelines
|
|
37
37
|
#TODO: can we make this not be based on size?
|
38
38
|
#it makes it harder to have streams as event sources (i.e. ranges).
|
39
39
|
#this class should only rely on Enumerable methods.
|
40
|
-
finished
|
41
|
-
|
42
|
-
if finished and finished_handler
|
43
|
-
@em.next_tick { finished_handler.call(self) }
|
44
|
-
end
|
40
|
+
finished! if (@num_finalised == @events.size)
|
45
41
|
end
|
46
42
|
end
|
47
43
|
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
module EmPipelines
|
2
|
+
class EventSource
|
3
|
+
def on_event(handler=nil, &block)
|
4
|
+
add_handlers!(event_handler, (handler || block))
|
5
|
+
end
|
6
|
+
|
7
|
+
def on_finished(handler=nil, &block)
|
8
|
+
add_handlers!(finished_handler, (handler || block))
|
9
|
+
end
|
10
|
+
|
11
|
+
protected
|
12
|
+
def event_handler
|
13
|
+
@event_handler ||= []
|
14
|
+
@event_handler
|
15
|
+
end
|
16
|
+
|
17
|
+
def finished_handler
|
18
|
+
@finished_handler ||= []
|
19
|
+
@finished_handler
|
20
|
+
end
|
21
|
+
|
22
|
+
def finished!
|
23
|
+
finished_handler.each{ |h| h.call(self) }
|
24
|
+
end
|
25
|
+
|
26
|
+
def event!(msg)
|
27
|
+
if event_handler.size == 1 then
|
28
|
+
event_handler.first.call(msg)
|
29
|
+
else
|
30
|
+
copies = a_copy_per_handler(msg)
|
31
|
+
event_handler.each do |h|
|
32
|
+
h.call(copies.pop)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
def a_copy_per_handler(msg)
|
39
|
+
msg_copies = event_handler.map { |x| msg.copy }
|
40
|
+
|
41
|
+
verify_copies = lambda do |m|
|
42
|
+
if msg_copies.all?(&:processed?) then
|
43
|
+
if msg_copies.any? { |m| m.state == :broken } then
|
44
|
+
msg.broken!
|
45
|
+
elsif msg_copies.any? { |m| m.state == :rejected } then
|
46
|
+
msg.rejected!
|
47
|
+
else
|
48
|
+
msg.consumed!
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
msg_copies.each do |copy|
|
54
|
+
copy.on_consumed(verify_copies)
|
55
|
+
copy.on_rejected(verify_copies)
|
56
|
+
copy.on_broken(verify_copies)
|
57
|
+
end
|
58
|
+
|
59
|
+
msg_copies.clone
|
60
|
+
end
|
61
|
+
|
62
|
+
def add_handlers!(handler_list, new_handlers)
|
63
|
+
to_add = new_handlers.is_a?(Enumerable) ? new_handlers : [new_handlers]
|
64
|
+
to_add.each { |h| handler_list << h }
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -1,8 +1,7 @@
|
|
1
|
-
require 'empipelines/
|
1
|
+
require 'empipelines/event_source'
|
2
2
|
|
3
3
|
module EmPipelines
|
4
|
-
class IOEventSource
|
5
|
-
include EventHandlers
|
4
|
+
class IOEventSource < EventSource
|
6
5
|
|
7
6
|
def initialize(em, file_path)
|
8
7
|
raise "File #{file_path} does not exist!" unless File.exists?(file_path)
|
@@ -15,7 +14,7 @@ module EmPipelines
|
|
15
14
|
|
16
15
|
wrapped_handler = BatchEventSource.new(@em, @file_path, events)
|
17
16
|
wrapped_handler.on_event(event_handler)
|
18
|
-
wrapped_handler.on_finished { |*ignored|
|
17
|
+
wrapped_handler.on_finished { |*ignored| finished! }
|
19
18
|
wrapped_handler.start!
|
20
19
|
end
|
21
20
|
end
|
data/lib/empipelines/message.rb
CHANGED
@@ -1,8 +1,12 @@
|
|
1
1
|
module EmPipelines
|
2
2
|
class Message
|
3
|
-
attr_reader :state
|
4
|
-
|
5
|
-
|
3
|
+
attr_reader :state, :co_id
|
4
|
+
|
5
|
+
@@count = 0
|
6
|
+
|
7
|
+
def initialize(base_hash={}, origin=nil)
|
8
|
+
@origin = origin
|
9
|
+
create_correlation_id!
|
6
10
|
backing_hash!(base_hash)
|
7
11
|
created!
|
8
12
|
end
|
@@ -28,15 +32,15 @@ module EmPipelines
|
|
28
32
|
end
|
29
33
|
|
30
34
|
def on_consumed(callback=nil, &callback_block)
|
31
|
-
@consumed_callback =
|
35
|
+
@consumed_callback = (callback || callback_block)
|
32
36
|
end
|
33
37
|
|
34
38
|
def on_rejected(callback=nil, &callback_block)
|
35
|
-
@rejected_callback =
|
39
|
+
@rejected_callback = (callback || callback_block)
|
36
40
|
end
|
37
|
-
|
38
|
-
def
|
39
|
-
@
|
41
|
+
|
42
|
+
def on_broken(callback=nil, &callback_block)
|
43
|
+
@broken_callback = (callback || callback_block)
|
40
44
|
end
|
41
45
|
|
42
46
|
def consumed!
|
@@ -50,11 +54,15 @@ module EmPipelines
|
|
50
54
|
@state = :rejected
|
51
55
|
invoke(@rejected_callback)
|
52
56
|
end
|
53
|
-
|
57
|
+
|
54
58
|
def broken!
|
55
59
|
check_if_mutation_allowed
|
56
|
-
@state = :
|
57
|
-
invoke(@
|
60
|
+
@state = :broken
|
61
|
+
invoke(@broken_callback)
|
62
|
+
end
|
63
|
+
|
64
|
+
def processed?
|
65
|
+
@state != :created
|
58
66
|
end
|
59
67
|
|
60
68
|
def as_hash
|
@@ -64,20 +72,33 @@ module EmPipelines
|
|
64
72
|
def payload
|
65
73
|
as_hash[:payload]
|
66
74
|
end
|
67
|
-
|
75
|
+
|
76
|
+
def copy
|
77
|
+
forked = Message.new(as_hash, self)
|
78
|
+
forked.on_broken(@broken_callback)
|
79
|
+
forked.on_rejected(@rejected_callback)
|
80
|
+
forked.on_consumed(@consumed_callback)
|
81
|
+
forked
|
82
|
+
end
|
83
|
+
|
68
84
|
def to_s
|
69
|
-
"#{self.class.name} state:#{@state} backing_hash:#{as_hash}"
|
85
|
+
"#{self.class.name} co_id:#{co_id} state:#{@state} backing_hash:#{as_hash}"
|
70
86
|
end
|
71
87
|
|
72
88
|
private
|
89
|
+
def create_correlation_id!
|
90
|
+
@@count += 1
|
91
|
+
suffix = @origin.nil? ? "@#{Process.pid}" : @origin.co_id
|
92
|
+
@co_id = "#{@@count}@#{suffix}"
|
93
|
+
end
|
73
94
|
|
74
95
|
def backing_hash!(other)
|
75
96
|
@backing_hash = symbolised(other)
|
76
97
|
end
|
77
|
-
|
98
|
+
|
78
99
|
def symbolised(raw_hash)
|
79
100
|
raw_hash.reduce({}) do |acc, (key, value)|
|
80
|
-
acc[key.to_s.to_sym] = value.is_a?(Hash) ? symbolised(value) : value
|
101
|
+
acc[key.to_s.to_sym] = value.is_a?(Hash) ? symbolised(value) : value
|
81
102
|
acc
|
82
103
|
end
|
83
104
|
end
|
@@ -85,11 +106,11 @@ module EmPipelines
|
|
85
106
|
def created!
|
86
107
|
@state = :created
|
87
108
|
end
|
88
|
-
|
109
|
+
|
89
110
|
def check_if_mutation_allowed
|
90
|
-
raise "Cannot mutate #{self}"
|
111
|
+
raise "Cannot mutate #{self}" if processed?
|
91
112
|
end
|
92
|
-
|
113
|
+
|
93
114
|
def invoke(callback)
|
94
115
|
callback.call(self) if callback
|
95
116
|
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
module EmPipelines
|
2
|
-
class PeriodicEventSource
|
2
|
+
class PeriodicEventSource < EventSource
|
3
3
|
#on finish!!!!
|
4
4
|
def initialize(em, name, interval_in_secs, &event_sourcing_code)
|
5
5
|
@em = em
|
@@ -16,14 +16,10 @@ module EmPipelines
|
|
16
16
|
end
|
17
17
|
end
|
18
18
|
|
19
|
-
def on_event(&handler)
|
20
|
-
@handler = handler
|
21
|
-
end
|
22
|
-
|
23
19
|
def tick!
|
24
20
|
event = @event_sourcing_code.call
|
25
21
|
|
26
|
-
|
22
|
+
event!(Message.new(:payload => event, :origin => @name)) if event
|
27
23
|
end
|
28
24
|
end
|
29
25
|
end
|
data/lib/empipelines.rb
CHANGED
@@ -1,14 +1,13 @@
|
|
1
1
|
module EmPipelines
|
2
|
-
VERSION = '0.2.
|
2
|
+
VERSION = '0.2.3'
|
3
3
|
end
|
4
4
|
|
5
5
|
require 'empipelines/message'
|
6
6
|
|
7
|
-
require 'empipelines/
|
7
|
+
require 'empipelines/event_source'
|
8
8
|
require 'empipelines/amqp_event_source'
|
9
9
|
require 'empipelines/batch_event_source'
|
10
10
|
require 'empipelines/io_event_source'
|
11
|
-
require 'empipelines/list_event_source'
|
12
11
|
require 'empipelines/periodic_event_source'
|
13
12
|
require 'empipelines/aggregated_event_source'
|
14
13
|
|
@@ -1,17 +1,15 @@
|
|
1
1
|
require 'empipelines/aggregated_event_source'
|
2
2
|
|
3
3
|
module EmPipelines
|
4
|
-
class EventSourceStub
|
5
|
-
|
6
|
-
|
7
|
-
def event!(contents)
|
4
|
+
class EventSourceStub < EventSource
|
5
|
+
def event_now!(contents)
|
8
6
|
raise 'not started' unless @started
|
9
|
-
|
7
|
+
event!(contents)
|
10
8
|
end
|
11
9
|
|
12
|
-
def
|
10
|
+
def finish_now!
|
13
11
|
raise 'not started' unless @started
|
14
|
-
|
12
|
+
finished!
|
15
13
|
end
|
16
14
|
|
17
15
|
def start!
|
@@ -40,11 +38,11 @@ module EmPipelines
|
|
40
38
|
|
41
39
|
aggregated.start!
|
42
40
|
|
43
|
-
source1.
|
44
|
-
source2.
|
45
|
-
source2.
|
46
|
-
source3.
|
47
|
-
source1.
|
41
|
+
source1.event_now! expected[0]
|
42
|
+
source2.event_now! expected[1]
|
43
|
+
source2.event_now! expected[2]
|
44
|
+
source3.event_now! expected[3]
|
45
|
+
source1.event_now! expected[4]
|
48
46
|
|
49
47
|
received.should ==(expected)
|
50
48
|
end
|
@@ -61,9 +59,9 @@ module EmPipelines
|
|
61
59
|
end
|
62
60
|
|
63
61
|
aggregated.start!
|
64
|
-
sources[2].
|
65
|
-
sources[1].
|
66
|
-
sources[0].
|
62
|
+
sources[2].finish_now!
|
63
|
+
sources[1].finish_now!
|
64
|
+
sources[0].finish_now!
|
67
65
|
|
68
66
|
has_finished.first.should be_true
|
69
67
|
end
|
@@ -78,8 +76,8 @@ module EmPipelines
|
|
78
76
|
end
|
79
77
|
|
80
78
|
aggregated.start!
|
81
|
-
sources[2].
|
82
|
-
sources[0].
|
79
|
+
sources[2].finish_now!
|
80
|
+
sources[0].finish_now!
|
83
81
|
end
|
84
82
|
end
|
85
83
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'empipelines/batch_event_source'
|
2
2
|
|
3
3
|
module EmPipelines
|
4
|
-
ShouldNotBeCalled = lambda { raise 'should not be called' }
|
4
|
+
ShouldNotBeCalled = lambda { |*x| raise 'should not be called' }
|
5
5
|
describe BatchEventSource do
|
6
6
|
|
7
7
|
let (:em) do
|
@@ -56,7 +56,7 @@ module EmPipelines
|
|
56
56
|
has_finished << true
|
57
57
|
end
|
58
58
|
|
59
|
-
source.on_event(
|
59
|
+
source.on_event(ShouldNotBeCalled)
|
60
60
|
|
61
61
|
source.start!
|
62
62
|
|
@@ -67,8 +67,6 @@ module EmPipelines
|
|
67
67
|
events = [1,2,3,4,5,6,7,8,9,10]
|
68
68
|
source = BatchEventSource.new(em, list_name, events)
|
69
69
|
|
70
|
-
source.on_event(&ShouldNotBeCalled)
|
71
|
-
|
72
70
|
count = 0
|
73
71
|
source.on_event do |e|
|
74
72
|
e.consumed! if (count=+1) > 1
|
@@ -0,0 +1,115 @@
|
|
1
|
+
require 'empipelines/event_source'
|
2
|
+
|
3
|
+
module EmPipelines
|
4
|
+
class StubEventSource < EventSource
|
5
|
+
def event_now!(message)
|
6
|
+
event!(message)
|
7
|
+
end
|
8
|
+
|
9
|
+
def finish_now!
|
10
|
+
finished!
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
describe EventSource do
|
15
|
+
context 'calling event handlers' do
|
16
|
+
it 'calls the handler when an message is to be processed' do
|
17
|
+
message = stub('message')
|
18
|
+
received = []
|
19
|
+
|
20
|
+
source = StubEventSource.new
|
21
|
+
source.on_event { |a| received << a }
|
22
|
+
source.event_now!(message)
|
23
|
+
received.should==([message])
|
24
|
+
end
|
25
|
+
|
26
|
+
it 'calls the finished handler when all events were processed' do
|
27
|
+
message = stub('message')
|
28
|
+
received = []
|
29
|
+
|
30
|
+
source = StubEventSource.new
|
31
|
+
source.on_finished { |a| received << a }
|
32
|
+
source.finish_now!
|
33
|
+
received.should==([source])
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'does not do anything if no handlers' do
|
37
|
+
StubEventSource.new.event_now!({})
|
38
|
+
StubEventSource.new.finish_now!
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
context 'multiple event handlers' do
|
43
|
+
let(:message1) { Message.new({:a => 1}) }
|
44
|
+
let(:message2) { Message.new({:b => 2}) }
|
45
|
+
|
46
|
+
def mark_as(state)
|
47
|
+
lambda {|m| m.send "#{state}!".to_sym }
|
48
|
+
end
|
49
|
+
|
50
|
+
def should_be_marked_as(desired, message)
|
51
|
+
undesired = [:broken, :consumed, :rejected] - [desired]
|
52
|
+
undesired.each { |u| message.should_not_receive("#{u}!".to_sym) }
|
53
|
+
message.should_receive("#{desired}!".to_sym)
|
54
|
+
message.should_receive(:copy).at_least(:once).and_return { Message.new({}) }
|
55
|
+
end
|
56
|
+
|
57
|
+
it 'supports multiple handlers for a single event'do
|
58
|
+
received = []
|
59
|
+
finished = []
|
60
|
+
|
61
|
+
source = StubEventSource.new
|
62
|
+
source.on_event([lambda {|m| received << [1, m.payload]}, lambda {|m| received << [2, m.payload]}])
|
63
|
+
source.on_event {|m| received << [3, m.payload]}
|
64
|
+
|
65
|
+
source.on_finished {|s| finished << [10, s]}
|
66
|
+
source.on_finished([lambda {|s| finished << [20, s]}, lambda {|s| finished << [30, s]}])
|
67
|
+
|
68
|
+
source.event_now!(message1)
|
69
|
+
source.event_now!(message2)
|
70
|
+
source.finish_now!
|
71
|
+
|
72
|
+
received.should ==([
|
73
|
+
[1, message1.payload],
|
74
|
+
[2, message1.payload],
|
75
|
+
[3, message1.payload],
|
76
|
+
[1, message2.payload],
|
77
|
+
[2, message2.payload],
|
78
|
+
[3, message2.payload]
|
79
|
+
])
|
80
|
+
|
81
|
+
finished.should ==([[10, source], [20, source], [30, source]])
|
82
|
+
end
|
83
|
+
|
84
|
+
it 'marks the message as consumed if all handlers consume' do
|
85
|
+
message = mock('message')
|
86
|
+
should_be_marked_as(:consumed, message)
|
87
|
+
|
88
|
+
source = StubEventSource.new
|
89
|
+
source.on_event([mark_as(:consumed), mark_as(:consumed), mark_as(:consumed)])
|
90
|
+
source.event_now!(message)
|
91
|
+
end
|
92
|
+
|
93
|
+
it 'marks the message as rejects if at least one handler rejects and all others consumed' do
|
94
|
+
message = mock('message')
|
95
|
+
should_be_marked_as(:rejected, message)
|
96
|
+
|
97
|
+
source = StubEventSource.new
|
98
|
+
source.on_event([mark_as(:consumed), mark_as(:consumed), mark_as(:rejected), mark_as(:consumed)])
|
99
|
+
source.event_now!(message)
|
100
|
+
end
|
101
|
+
|
102
|
+
it 'marks message as broken if at least one handler marks as broken, regardless of others' do
|
103
|
+
message = mock('message')
|
104
|
+
should_be_marked_as(:broken, message)
|
105
|
+
|
106
|
+
source = StubEventSource.new
|
107
|
+
source.on_event([mark_as(:consumed), mark_as(:consumed), mark_as(:rejected), mark_as(:consumed), mark_as(:broken)])
|
108
|
+
source.event_now!(message)
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
context 'flow control' do
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
@@ -2,8 +2,8 @@ require 'empipelines/message'
|
|
2
2
|
|
3
3
|
module EmPipelines
|
4
4
|
describe Message do
|
5
|
-
context
|
6
|
-
it
|
5
|
+
context 'mostly behaves like a hashmap' do
|
6
|
+
it 'stores values under symbolised keys' do
|
7
7
|
original_hash = {:a => 1, :b => 2}
|
8
8
|
|
9
9
|
message = Message.new(original_hash)
|
@@ -13,23 +13,23 @@ module EmPipelines
|
|
13
13
|
message[:doesntexist].should ==(original_hash[:doesntexist])
|
14
14
|
end
|
15
15
|
|
16
|
-
it
|
16
|
+
it 'symbolises keys of all maps in the message' do
|
17
17
|
message = Message.new({
|
18
18
|
:a => 1,
|
19
|
-
|
19
|
+
'b' => 2,
|
20
20
|
3 => 3 ,
|
21
|
-
|
22
|
-
|
23
|
-
|
21
|
+
'd' => {
|
22
|
+
'd1' => {'e' => 5},
|
23
|
+
'd2' => nil}
|
24
24
|
})
|
25
25
|
message[:a].should ==(1)
|
26
26
|
message[:b].should ==(2)
|
27
|
-
message[
|
27
|
+
message['3'.to_sym].should ==(3)
|
28
28
|
message[:d][:d1][:e].should ==(5)
|
29
29
|
message[:d][:d2].should be_nil
|
30
30
|
end
|
31
|
-
|
32
|
-
it
|
31
|
+
|
32
|
+
it 'allows for values to be CRUD' do
|
33
33
|
original_hash = {:a => 1, :b => 2, :c => 0}
|
34
34
|
|
35
35
|
message = Message.new(original_hash)
|
@@ -41,10 +41,10 @@ module EmPipelines
|
|
41
41
|
message[:a].should ==(666)
|
42
42
|
message[:b].should be_nil
|
43
43
|
message[:c].should ==(original_hash[:c])
|
44
|
-
message[:z].should ==(999)
|
44
|
+
message[:z].should ==(999)
|
45
45
|
end
|
46
46
|
|
47
|
-
it
|
47
|
+
it 'can be merged with a map, symbolising keys' do
|
48
48
|
original = Message.new({'a' => 1})
|
49
49
|
original.merge!({'b' => 2})
|
50
50
|
|
@@ -53,74 +53,94 @@ module EmPipelines
|
|
53
53
|
end
|
54
54
|
end
|
55
55
|
|
56
|
-
context
|
56
|
+
context 'message status handlers' do
|
57
57
|
|
58
|
-
let (:handler_that_should_never_be_called) { lambda { raise
|
58
|
+
let (:handler_that_should_never_be_called) { lambda { raise 'This shouldnt happen' } }
|
59
59
|
|
60
|
-
it
|
60
|
+
it 'doesnt do anything if no state callback specified' do
|
61
61
|
Message.new.consumed!
|
62
62
|
Message.new.rejected!
|
63
|
-
Message.new.broken!
|
63
|
+
Message.new.broken!
|
64
|
+
end
|
65
|
+
|
66
|
+
it 'is possible to override a handler' do
|
67
|
+
origin = Message.new
|
68
|
+
origin.on_broken(handler_that_should_never_be_called)
|
69
|
+
origin.on_rejected(handler_that_should_never_be_called)
|
70
|
+
origin.on_consumed(handler_that_should_never_be_called)
|
71
|
+
|
72
|
+
consume = origin.copy
|
73
|
+
consume.on_consumed {}
|
74
|
+
|
75
|
+
reject = origin.copy
|
76
|
+
reject.on_rejected {}
|
77
|
+
|
78
|
+
broken = origin.copy
|
79
|
+
broken.on_broken {}
|
80
|
+
|
81
|
+
consume.consumed!
|
82
|
+
broken.broken!
|
83
|
+
reject.rejected!
|
64
84
|
end
|
65
85
|
|
66
|
-
it
|
86
|
+
it 'is possible to reject a message if broken'do
|
67
87
|
called = []
|
68
|
-
|
69
|
-
message = Message.new
|
70
|
-
message.
|
88
|
+
|
89
|
+
message = Message.new
|
90
|
+
message.on_broken do |msg|
|
71
91
|
called << msg
|
72
92
|
end
|
73
|
-
|
74
|
-
message.on_rejected(handler_that_should_never_be_called)
|
93
|
+
|
94
|
+
message.on_rejected(handler_that_should_never_be_called)
|
75
95
|
message.on_consumed(handler_that_should_never_be_called)
|
76
96
|
|
77
97
|
message.broken!
|
78
|
-
|
98
|
+
|
79
99
|
called.should==([message])
|
80
100
|
end
|
81
|
-
|
82
|
-
it
|
101
|
+
|
102
|
+
it 'is possible to reject a message if consumer cant handle it' do
|
83
103
|
called = []
|
84
|
-
|
85
|
-
message = Message.new
|
104
|
+
|
105
|
+
message = Message.new
|
86
106
|
message.on_rejected do |msg|
|
87
107
|
called << msg
|
88
108
|
end
|
89
|
-
|
90
|
-
message.
|
109
|
+
|
110
|
+
message.on_broken(handler_that_should_never_be_called)
|
91
111
|
message.on_consumed(handler_that_should_never_be_called)
|
92
112
|
|
93
113
|
message.rejected!
|
94
|
-
|
114
|
+
|
95
115
|
called.should==([message])
|
96
116
|
end
|
97
117
|
|
98
|
-
it
|
118
|
+
it 'is possible to mark a message as consumed' do
|
99
119
|
called = []
|
100
|
-
|
101
|
-
message = Message.new
|
120
|
+
|
121
|
+
message = Message.new
|
102
122
|
message.on_consumed do |msg|
|
103
123
|
called << msg
|
104
124
|
end
|
105
|
-
|
106
|
-
message.
|
125
|
+
|
126
|
+
message.on_broken(handler_that_should_never_be_called)
|
107
127
|
message.on_rejected(handler_that_should_never_be_called)
|
108
128
|
|
109
129
|
message.consumed!
|
110
|
-
|
130
|
+
|
111
131
|
called.should==([message])
|
112
132
|
end
|
113
|
-
|
114
|
-
it
|
133
|
+
|
134
|
+
it 'is not possible to change a message after marking as consumed or rejected' do
|
115
135
|
read = lambda { |m| m[:some_key] }
|
116
136
|
mutate = lambda { |m| m[:some_key] = :some_value }
|
117
|
-
|
137
|
+
|
118
138
|
consumed = Message.new
|
119
139
|
consumed.consumed!
|
120
|
-
|
140
|
+
|
121
141
|
rejected = Message.new
|
122
142
|
rejected.rejected!
|
123
|
-
|
143
|
+
|
124
144
|
broken = Message.new
|
125
145
|
broken.broken!
|
126
146
|
|
@@ -132,5 +152,104 @@ module EmPipelines
|
|
132
152
|
lambda{ mutate.call(broken) }.should raise_error
|
133
153
|
end
|
134
154
|
end
|
155
|
+
|
156
|
+
context 'cloning messages' do
|
157
|
+
it 'copys a message with equal initial state' do
|
158
|
+
origin = Message.new({:a => 1})
|
159
|
+
copy = origin.copy
|
160
|
+
|
161
|
+
origin.as_hash.should ==(copy.as_hash)
|
162
|
+
end
|
163
|
+
|
164
|
+
it 'copys a message with equal handlers' do
|
165
|
+
origin = Message.new({:a => 1})
|
166
|
+
|
167
|
+
consumed = []
|
168
|
+
rejected = []
|
169
|
+
broken = []
|
170
|
+
|
171
|
+
origin.on_consumed { |m| consumed << m}
|
172
|
+
origin.on_rejected { |m| rejected << m}
|
173
|
+
origin.on_broken { |m| broken << m}
|
174
|
+
|
175
|
+
copy1 = origin.copy
|
176
|
+
copy2 = origin.copy
|
177
|
+
|
178
|
+
origin.consumed!
|
179
|
+
copy1.broken!
|
180
|
+
copy2.rejected!
|
181
|
+
|
182
|
+
consumed.should ==([origin])
|
183
|
+
rejected.should ==([copy2])
|
184
|
+
broken.should ==([copy1])
|
185
|
+
end
|
186
|
+
|
187
|
+
it 'makes the messages contents independent' do
|
188
|
+
origin = Message.new({:a => 1})
|
189
|
+
copy = origin.copy
|
190
|
+
origin[:b] = 2
|
191
|
+
copy[:c] = 3
|
192
|
+
|
193
|
+
origin[:c].should be_nil
|
194
|
+
copy[:c].should_not be_nil
|
195
|
+
|
196
|
+
origin[:b].should_not be_nil
|
197
|
+
copy[:b].should be_nil
|
198
|
+
end
|
199
|
+
|
200
|
+
it 'makes the messages state independent' do
|
201
|
+
origin = Message.new({:a => 1})
|
202
|
+
copy = origin.copy
|
203
|
+
|
204
|
+
origin.broken!
|
205
|
+
copy.consumed!
|
206
|
+
|
207
|
+
origin.state.should ==(:broken)
|
208
|
+
copy.state.should ==(:consumed)
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
context 'generating a correlation id' do
|
213
|
+
it 'creates a different CoId for each sequential message' do
|
214
|
+
m1, m2, m3 = Message.new, Message.new, Message.new
|
215
|
+
|
216
|
+
m1.co_id.should_not ==(m2.co_id)
|
217
|
+
m1.co_id.should_not ==(m3.co_id)
|
218
|
+
m2.co_id.should_not ==(m1.co_id)
|
219
|
+
m2.co_id.should_not ==(m3.co_id)
|
220
|
+
m3.co_id.should_not ==(m2.co_id)
|
221
|
+
m3.co_id.should_not ==(m1.co_id)
|
222
|
+
end
|
223
|
+
|
224
|
+
it 'includes the process id on CoId so that multiple instances have different ids' do
|
225
|
+
Message.new.co_id.should match(/#{Process.pid}/)
|
226
|
+
end
|
227
|
+
|
228
|
+
it 'does not change CoId with state chages' do
|
229
|
+
m = Message.new
|
230
|
+
old_id = m.co_id
|
231
|
+
|
232
|
+
m.merge!({:some => :thing})
|
233
|
+
merged_id = m.co_id
|
234
|
+
|
235
|
+
m.consumed!
|
236
|
+
consumed_id = m.co_id
|
237
|
+
|
238
|
+
[merged_id, consumed_id].should ==([old_id, old_id])
|
239
|
+
end
|
240
|
+
|
241
|
+
it 'a copied message has a different, yet related, CoId from its origin' do
|
242
|
+
origin = Message.new
|
243
|
+
copied = origin.copy
|
244
|
+
grandcopied = copied.copy
|
245
|
+
|
246
|
+
origin.co_id.should_not ==(copied.co_id)
|
247
|
+
origin.co_id.should_not ==(grandcopied.co_id)
|
248
|
+
copied.co_id.should_not ==(grandcopied.co_id)
|
249
|
+
|
250
|
+
copied.co_id.should match(/#{origin.co_id}/)
|
251
|
+
grandcopied.co_id.should match(/#{copied.co_id}/)
|
252
|
+
end
|
253
|
+
end
|
135
254
|
end
|
136
255
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: empipelines
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2012-01-02 00:00:00.000000000Z
|
14
14
|
dependencies: []
|
15
15
|
description: Simple Event Handling Pipeline Architecture for EventMachine
|
16
16
|
email: pcalcado+empipelines@gmail.com
|
@@ -33,10 +33,9 @@ files:
|
|
33
33
|
- lib/empipelines/aggregated_event_source.rb
|
34
34
|
- lib/empipelines/amqp_event_source.rb
|
35
35
|
- lib/empipelines/batch_event_source.rb
|
36
|
-
- lib/empipelines/event_handlers.rb
|
37
36
|
- lib/empipelines/event_pipeline.rb
|
37
|
+
- lib/empipelines/event_source.rb
|
38
38
|
- lib/empipelines/io_event_source.rb
|
39
|
-
- lib/empipelines/list_event_source.rb
|
40
39
|
- lib/empipelines/message.rb
|
41
40
|
- lib/empipelines/periodic_event_source.rb
|
42
41
|
- lib/empipelines/pipeline.rb
|
@@ -45,9 +44,9 @@ files:
|
|
45
44
|
- unit/empipelines/batch_event_source_spec.rb
|
46
45
|
- unit/empipelines/empty_io_event_source.dat
|
47
46
|
- unit/empipelines/event_pipeline_spec.rb
|
47
|
+
- unit/empipelines/event_source_spec.rb
|
48
48
|
- unit/empipelines/io_event_source.dat
|
49
49
|
- unit/empipelines/io_event_source_spec.rb
|
50
|
-
- unit/empipelines/list_event_source_spec.rb
|
51
50
|
- unit/empipelines/message_spec.rb
|
52
51
|
- unit/empipelines/periodic_event_source_spec.rb
|
53
52
|
- unit/empipelines/pipeline_spec.rb
|
@@ -68,7 +67,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
68
67
|
version: '0'
|
69
68
|
segments:
|
70
69
|
- 0
|
71
|
-
hash:
|
70
|
+
hash: 2169333879126328078
|
72
71
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
73
72
|
none: false
|
74
73
|
requirements:
|
@@ -1,20 +0,0 @@
|
|
1
|
-
module EmPipelines
|
2
|
-
module EventHandlers
|
3
|
-
def on_event(event_handler=nil, &block)
|
4
|
-
@event_handler = block_given? ? block : event_handler
|
5
|
-
end
|
6
|
-
|
7
|
-
def on_finished(batch_finished_handler=nil, &block)
|
8
|
-
@finished_handler = block_given? ? block : batch_finished_handler
|
9
|
-
end
|
10
|
-
|
11
|
-
protected
|
12
|
-
def finished_handler
|
13
|
-
@finished_handler
|
14
|
-
end
|
15
|
-
|
16
|
-
def event_handler
|
17
|
-
@event_handler
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
@@ -1,17 +0,0 @@
|
|
1
|
-
module EmPipelines
|
2
|
-
class ListEventSource
|
3
|
-
def initialize(events)
|
4
|
-
@events = events
|
5
|
-
end
|
6
|
-
|
7
|
-
def start!
|
8
|
-
@events.each do |e|
|
9
|
-
@handler.call({:payload => e})
|
10
|
-
end
|
11
|
-
end
|
12
|
-
|
13
|
-
def on_event(&handler)
|
14
|
-
@handler = handler
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
@@ -1,17 +0,0 @@
|
|
1
|
-
require 'empipelines/list_event_source'
|
2
|
-
|
3
|
-
module EmPipelines
|
4
|
-
describe ListEventSource do
|
5
|
-
it 'sends each element of the list to the handler' do
|
6
|
-
items = [1, 2, 3, 4, 5, 6]
|
7
|
-
expected_messages = items.map { |i| {:payload => i} }
|
8
|
-
received_messages = []
|
9
|
-
|
10
|
-
source = ListEventSource.new(items)
|
11
|
-
source.on_event { |msg| received_messages << msg}
|
12
|
-
source.start!
|
13
|
-
|
14
|
-
received_messages.should eql(expected_messages)
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|