batsir 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +5 -0
- data/.rspec +3 -0
- data/.travis.yml +10 -0
- data/Gemfile +22 -0
- data/LICENSE.txt +20 -0
- data/README.md +79 -0
- data/Rakefile +55 -0
- data/VERSION +1 -0
- data/batsir.gemspec +104 -0
- data/batsir.png +0 -0
- data/lib/batsir.rb +73 -0
- data/lib/batsir/acceptors/acceptor.rb +45 -0
- data/lib/batsir/acceptors/amqp_acceptor.rb +19 -0
- data/lib/batsir/amqp.rb +45 -0
- data/lib/batsir/chain.rb +33 -0
- data/lib/batsir/config.rb +34 -0
- data/lib/batsir/dsl/dsl_mappings.rb +96 -0
- data/lib/batsir/filter.rb +12 -0
- data/lib/batsir/filter_queue.rb +30 -0
- data/lib/batsir/logo.rb +36 -0
- data/lib/batsir/notifiers/amqp_notifier.rb +16 -0
- data/lib/batsir/notifiers/notifier.rb +39 -0
- data/lib/batsir/registry.rb +15 -0
- data/lib/batsir/stage.rb +94 -0
- data/lib/batsir/stage_worker.rb +86 -0
- data/lib/batsir/transformers/field_transformer.rb +40 -0
- data/lib/batsir/transformers/json_input_transformer.rb +9 -0
- data/lib/batsir/transformers/json_output_transformer.rb +9 -0
- data/lib/batsir/transformers/transformer.rb +15 -0
- data/spec/batsir/acceptors/acceptor_spec.rb +136 -0
- data/spec/batsir/acceptors/amqp_acceptor_spec.rb +169 -0
- data/spec/batsir/chain_spec.rb +31 -0
- data/spec/batsir/dsl/chain_mapping_spec.rb +117 -0
- data/spec/batsir/dsl/stage_mapping_spec.rb +435 -0
- data/spec/batsir/filter_queue_spec.rb +74 -0
- data/spec/batsir/filter_spec.rb +12 -0
- data/spec/batsir/notifiers/amqp_notifier_spec.rb +117 -0
- data/spec/batsir/notifiers/notifier_spec.rb +73 -0
- data/spec/batsir/stage_spec.rb +678 -0
- data/spec/batsir/stage_worker_spec.rb +128 -0
- data/spec/batsir/support/bunny_mocks.rb +62 -0
- data/spec/batsir/support/mock_filters.rb +43 -0
- data/spec/batsir/transformers/field_transformer_spec.rb +73 -0
- data/spec/batsir/transformers/json_input_transformer_spec.rb +22 -0
- data/spec/batsir/transformers/json_output_transformer_spec.rb +18 -0
- data/spec/batsir/transformers/transformer_spec.rb +22 -0
- data/spec/spec_helper.rb +22 -0
- metadata +220 -0
@@ -0,0 +1,86 @@
|
|
1
|
+
require 'sidekiq'
|
2
|
+
|
3
|
+
module Batsir
|
4
|
+
module StageWorker
|
5
|
+
attr_accessor :filter_queue
|
6
|
+
|
7
|
+
def self.included(base)
|
8
|
+
Registry.register(base.stage_name, base)
|
9
|
+
base.initialize_filter_queue
|
10
|
+
end
|
11
|
+
|
12
|
+
def perform(message)
|
13
|
+
execute(message)
|
14
|
+
end
|
15
|
+
|
16
|
+
def execute(message)
|
17
|
+
return false unless @filter_queue
|
18
|
+
@filter_queue.filters.each do |filter|
|
19
|
+
message = filter.execute(message)
|
20
|
+
return false if message.nil?
|
21
|
+
end
|
22
|
+
@filter_queue.notifiers.each do |notifier|
|
23
|
+
notifier.notify(message)
|
24
|
+
end
|
25
|
+
true
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.compile_from(stage)
|
29
|
+
code = <<-EOF
|
30
|
+
class #{stage.name.capitalize.gsub(' ','')}Worker
|
31
|
+
def self.stage_name
|
32
|
+
"#{stage.name}"
|
33
|
+
end
|
34
|
+
|
35
|
+
def initialize
|
36
|
+
@filter_queue = self.class.filter_queue
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.filter_queue
|
40
|
+
@filter_queue
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.initialize_filter_queue
|
44
|
+
@filter_queue = Batsir::FilterQueue.new
|
45
|
+
EOF
|
46
|
+
|
47
|
+
stage.filter_declarations.each do |filter_declaration|
|
48
|
+
code << <<-EOF
|
49
|
+
@filter_queue.add #{filter_declaration.filter.to_s}.new(#{filter_declaration.options.to_s})
|
50
|
+
EOF
|
51
|
+
end
|
52
|
+
|
53
|
+
stage.notifiers.each do |notifier, options_set|
|
54
|
+
options_set.each do |options|
|
55
|
+
code << <<-EOF
|
56
|
+
notifier = #{notifier.to_s}.new(#{options.to_s})
|
57
|
+
EOF
|
58
|
+
|
59
|
+
if stage.notifier_transformers.any?
|
60
|
+
stage.notifier_transformers.each do |transformer_declaration|
|
61
|
+
code << <<-EOF
|
62
|
+
notifier.add_transformer #{transformer_declaration.transformer}.new(#{transformer_declaration.options.to_s})
|
63
|
+
EOF
|
64
|
+
end
|
65
|
+
else
|
66
|
+
code << <<-EOF
|
67
|
+
notifier.add_transformer Batsir::Transformers::JSONOutputTransformer.new
|
68
|
+
EOF
|
69
|
+
end
|
70
|
+
code << <<-EOF
|
71
|
+
@filter_queue.add_notifier notifier
|
72
|
+
EOF
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
code << <<-EOF
|
77
|
+
end
|
78
|
+
|
79
|
+
include Sidekiq::Worker
|
80
|
+
include Batsir::StageWorker
|
81
|
+
end
|
82
|
+
EOF
|
83
|
+
code
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Batsir
|
2
|
+
module Transformers
|
3
|
+
class FieldTransformer < Transformer
|
4
|
+
attr_accessor :fields
|
5
|
+
|
6
|
+
def fields
|
7
|
+
@fields ||= {}
|
8
|
+
end
|
9
|
+
|
10
|
+
def fields=(hash)
|
11
|
+
@fields = {}
|
12
|
+
hash.each do |k, v|
|
13
|
+
@fields[k.to_sym] = v.to_sym
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def transform(message)
|
18
|
+
fields = self.fields
|
19
|
+
if fields.any? && message.respond_to?(:keys)
|
20
|
+
symbolized_message_keys = {}
|
21
|
+
message.keys.each do |key|
|
22
|
+
symbolized_message_keys[key.to_sym] = key
|
23
|
+
end
|
24
|
+
|
25
|
+
fields_to_remove = symbolized_message_keys.keys - fields.keys - fields.values
|
26
|
+
|
27
|
+
fields.each do |new, old|
|
28
|
+
message[new] = message.delete(symbolized_message_keys[old])
|
29
|
+
end
|
30
|
+
|
31
|
+
fields_to_remove.each do |field|
|
32
|
+
message.delete(symbolized_message_keys[field])
|
33
|
+
end
|
34
|
+
end
|
35
|
+
message
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,136 @@
|
|
1
|
+
require File.join( File.dirname(__FILE__), "..", "..", "spec_helper" )
|
2
|
+
|
3
|
+
describe Batsir::Acceptors::Acceptor do
|
4
|
+
let( :acceptor_class ) {
|
5
|
+
Batsir::Acceptors::Acceptor
|
6
|
+
}
|
7
|
+
|
8
|
+
it "should be a Celluloid actor" do
|
9
|
+
acceptor_class.ancestors.should include Celluloid
|
10
|
+
end
|
11
|
+
|
12
|
+
it "should be possible to set the stage name for the acceptor" do
|
13
|
+
acceptor = acceptor_class.new
|
14
|
+
stage_name = "some stage"
|
15
|
+
acceptor.stage_name = stage_name
|
16
|
+
acceptor.stage_name.should == stage_name
|
17
|
+
end
|
18
|
+
|
19
|
+
it "should have a #start method" do
|
20
|
+
acceptor_class.instance_methods.map{|im| im.to_s}.should include "start"
|
21
|
+
end
|
22
|
+
|
23
|
+
it "should have an #start_filter_chain method" do
|
24
|
+
acceptor_class.instance_methods.map{|im| im.to_s}.should include "start_filter_chain"
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should be possible to initialize an Acceptor with on options hash" do
|
28
|
+
acceptor = acceptor_class.new({})
|
29
|
+
acceptor.should_not be_nil
|
30
|
+
end
|
31
|
+
|
32
|
+
it "should look up a worker class when the #start_filter_chain method is called" do
|
33
|
+
acceptor = acceptor_class.new
|
34
|
+
stage_name = "some stage"
|
35
|
+
acceptor.stage_name = stage_name
|
36
|
+
Batsir::Registry.should_receive(:get).with(stage_name)
|
37
|
+
acceptor.start_filter_chain({})
|
38
|
+
end
|
39
|
+
|
40
|
+
it "should have a transformer_queue" do
|
41
|
+
acceptor = acceptor_class.new
|
42
|
+
acceptor.transformer_queue.should_not be_nil
|
43
|
+
end
|
44
|
+
|
45
|
+
it "should initially have an empty transformer_queue" do
|
46
|
+
acceptor = acceptor_class.new
|
47
|
+
acceptor.transformer_queue.should_not be_nil
|
48
|
+
acceptor.transformer_queue.should be_empty
|
49
|
+
end
|
50
|
+
|
51
|
+
it "should be possible to add a transformer to the transformer_queue" do
|
52
|
+
transformer = :transformer
|
53
|
+
|
54
|
+
acceptor = acceptor_class.new
|
55
|
+
acceptor.add_transformer transformer
|
56
|
+
|
57
|
+
acceptor.transformer_queue.should_not be_empty
|
58
|
+
acceptor.transformer_queue.size.should == 1
|
59
|
+
acceptor.transformer_queue.first.should == :transformer
|
60
|
+
end
|
61
|
+
|
62
|
+
it "should be possible to add a transformer multiple times" do
|
63
|
+
transformer = :transformer
|
64
|
+
|
65
|
+
acceptor = acceptor_class.new
|
66
|
+
acceptor.add_transformer transformer
|
67
|
+
acceptor.add_transformer transformer
|
68
|
+
|
69
|
+
acceptor.transformer_queue.should_not be_empty
|
70
|
+
acceptor.transformer_queue.size.should == 2
|
71
|
+
end
|
72
|
+
|
73
|
+
it "should be possible to set a cancellator" do
|
74
|
+
cancellator = :cancel
|
75
|
+
acceptor = acceptor_class.new
|
76
|
+
acceptor.cancellator = cancellator
|
77
|
+
acceptor.cancellator.should == cancellator
|
78
|
+
end
|
79
|
+
|
80
|
+
it "should be possible to set a cancellator in the constructor using a hash" do
|
81
|
+
cancellator = :cancel
|
82
|
+
acceptor = acceptor_class.new(:cancellator => cancellator)
|
83
|
+
acceptor.cancellator.should == cancellator
|
84
|
+
end
|
85
|
+
|
86
|
+
it "should call the #perform_async on the worker class when #start_filter_chain is called" do
|
87
|
+
class MockWorker
|
88
|
+
def self.stage_name
|
89
|
+
"mock_stage"
|
90
|
+
end
|
91
|
+
|
92
|
+
def self.initialize_filter_queue
|
93
|
+
end
|
94
|
+
|
95
|
+
def self.perform_async(*args)
|
96
|
+
end
|
97
|
+
|
98
|
+
include Batsir::StageWorker
|
99
|
+
end
|
100
|
+
|
101
|
+
acceptor = acceptor_class.new
|
102
|
+
stage_name = "some stage"
|
103
|
+
|
104
|
+
acceptor.stage_name = stage_name
|
105
|
+
|
106
|
+
Batsir::Registry.register(stage_name, MockWorker)
|
107
|
+
MockWorker.should_receive(:perform_async)
|
108
|
+
acceptor.start_filter_chain({})
|
109
|
+
end
|
110
|
+
|
111
|
+
it "should call #transform on the acceptor transformers" do
|
112
|
+
class MockTransformer < Batsir::Transformers::Transformer
|
113
|
+
def transform(message)
|
114
|
+
@@transformed ||= 0
|
115
|
+
@@transformed += 1
|
116
|
+
end
|
117
|
+
|
118
|
+
def self.transformed
|
119
|
+
@@transformed ||= 0
|
120
|
+
@@transformed
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
acceptor = acceptor_class.new
|
125
|
+
stage_name = "some stage"
|
126
|
+
|
127
|
+
acceptor.stage_name = stage_name
|
128
|
+
acceptor.add_transformer MockTransformer.new
|
129
|
+
|
130
|
+
MockTransformer.transformed.should == 0
|
131
|
+
|
132
|
+
acceptor.start_filter_chain({})
|
133
|
+
|
134
|
+
MockTransformer.transformed.should == 1
|
135
|
+
end
|
136
|
+
end
|
@@ -0,0 +1,169 @@
|
|
1
|
+
require File.join( File.dirname(__FILE__), "..", "..", "spec_helper" )
|
2
|
+
|
3
|
+
describe Batsir::Acceptors::AMQPAcceptor do
|
4
|
+
|
5
|
+
let(:acceptor_class){
|
6
|
+
Batsir::Acceptors::AMQPAcceptor
|
7
|
+
}
|
8
|
+
|
9
|
+
context "With respect to setting options" do
|
10
|
+
|
11
|
+
it "should be a Batsir::Acceptors::Acceptor" do
|
12
|
+
acceptor_class.ancestors.should include Batsir::Acceptors::Acceptor
|
13
|
+
end
|
14
|
+
|
15
|
+
it "should be possible to set the queue on which to listen" do
|
16
|
+
acceptor = acceptor_class.new(:queue => :queue)
|
17
|
+
acceptor.queue.should == :queue
|
18
|
+
end
|
19
|
+
|
20
|
+
it "should be possible to set the host of the amqp broker" do
|
21
|
+
acceptor = acceptor_class.new(:host => 'localhost')
|
22
|
+
acceptor.host.should == 'localhost'
|
23
|
+
end
|
24
|
+
|
25
|
+
it "should be possible to set the port of the amqp broker" do
|
26
|
+
acceptor = acceptor_class.new(:port => 1234)
|
27
|
+
acceptor.port.should == 1234
|
28
|
+
end
|
29
|
+
|
30
|
+
it "should be possible to set the username with which to connect to the broker" do
|
31
|
+
acceptor = acceptor_class.new(:username => 'some_user')
|
32
|
+
acceptor.username.should == 'some_user'
|
33
|
+
end
|
34
|
+
|
35
|
+
it "should be possible to set the password with which to connect to the broker" do
|
36
|
+
acceptor = acceptor_class.new(:password => 'password')
|
37
|
+
acceptor.password.should == 'password'
|
38
|
+
end
|
39
|
+
|
40
|
+
it "should be possible to set the vhost to use on the broker" do
|
41
|
+
acceptor = acceptor_class.new(:vhost => '/vhost')
|
42
|
+
acceptor.vhost.should == '/vhost'
|
43
|
+
end
|
44
|
+
|
45
|
+
it "should be possible to set the exchange to use on the broker" do
|
46
|
+
acceptor = acceptor_class.new(:exchange => 'amq.direct')
|
47
|
+
acceptor.exchange.should == 'amq.direct'
|
48
|
+
end
|
49
|
+
|
50
|
+
it "should default to amqp://guest:guest@localhost:5672/ with direct exchange on vhost ''" do
|
51
|
+
acceptor = acceptor_class.new(:queue => :somequeue)
|
52
|
+
acceptor.queue.should == :somequeue
|
53
|
+
acceptor.host.should == 'localhost'
|
54
|
+
acceptor.port.should == 5672
|
55
|
+
acceptor.username.should == 'guest'
|
56
|
+
acceptor.password.should == 'guest'
|
57
|
+
acceptor.vhost.should == '/'
|
58
|
+
acceptor.exchange.should == 'amq.direct'
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
context "with respect to starting the acceptor" do
|
63
|
+
def new_acceptor(options = {})
|
64
|
+
acceptor_class.new(options)
|
65
|
+
end
|
66
|
+
|
67
|
+
it "should connect to the configured host" do
|
68
|
+
acceptor = new_acceptor(:host => 'somehost')
|
69
|
+
acceptor.start
|
70
|
+
instance = Bunny.instance
|
71
|
+
instance.options[:host].should == 'somehost'
|
72
|
+
end
|
73
|
+
|
74
|
+
it "should connect to the configured port" do
|
75
|
+
acceptor = new_acceptor(:port => 1234)
|
76
|
+
acceptor.start
|
77
|
+
instance = Bunny.instance
|
78
|
+
instance.options[:port].should == 1234
|
79
|
+
end
|
80
|
+
|
81
|
+
it "should connect with the configured username" do
|
82
|
+
acceptor = new_acceptor(:username => 'user')
|
83
|
+
acceptor.start
|
84
|
+
instance = Bunny.instance
|
85
|
+
instance.options[:user].should == 'user'
|
86
|
+
end
|
87
|
+
|
88
|
+
it "should connect with the configured password" do
|
89
|
+
acceptor = new_acceptor(:password => 'pass')
|
90
|
+
acceptor.start
|
91
|
+
instance = Bunny.instance
|
92
|
+
instance.options[:pass].should == 'pass'
|
93
|
+
end
|
94
|
+
|
95
|
+
it "should connect to the configured vhost" do
|
96
|
+
acceptor = new_acceptor(:vhost => '/vhost')
|
97
|
+
acceptor.start
|
98
|
+
instance = Bunny.instance
|
99
|
+
instance.options[:vhost].should == '/vhost'
|
100
|
+
end
|
101
|
+
|
102
|
+
it "should declare the configured exchange" do
|
103
|
+
acceptor = new_acceptor(:exchange => 'some_exchange')
|
104
|
+
acceptor.start
|
105
|
+
instance = Bunny.instance
|
106
|
+
instance.exchange.name.should == 'some_exchange'
|
107
|
+
end
|
108
|
+
|
109
|
+
it "should bind the configured exchange to the queue" do
|
110
|
+
acceptor = new_acceptor(:exchange => 'some_exchange', :queue => :queue)
|
111
|
+
acceptor.start
|
112
|
+
instance = Bunny.instance
|
113
|
+
queue = instance.queues[:queue]
|
114
|
+
queue.bound_exchange.name.should == 'some_exchange'
|
115
|
+
queue.bound_key.should == :queue
|
116
|
+
end
|
117
|
+
|
118
|
+
it "should start listening on the configured queue" do
|
119
|
+
acceptor = new_acceptor(:queue => :some_queue)
|
120
|
+
acceptor.start
|
121
|
+
instance = Bunny.instance
|
122
|
+
instance.queues.size.should == 1
|
123
|
+
instance.queues.keys.should include :some_queue
|
124
|
+
end
|
125
|
+
|
126
|
+
it "should initialize the subscription with the acceptor's cancellator" do
|
127
|
+
cancellator = :cancellator
|
128
|
+
acceptor = new_acceptor(:queue => :some_queue, :cancellator => cancellator)
|
129
|
+
acceptor.start
|
130
|
+
instance = Bunny.instance
|
131
|
+
puts instance.queues[:some_queue].arguments.first[:cancellator].should == cancellator
|
132
|
+
end
|
133
|
+
|
134
|
+
it "should call the #start_filter_chain method when a message is received" do
|
135
|
+
acceptor = new_acceptor(:queue => :some_queue)
|
136
|
+
|
137
|
+
# Because acceptor is a Celluloid Actor, it is not possible to define a method
|
138
|
+
# on the instance directly, because you actually hold a reference to a
|
139
|
+
# ActorProxy.
|
140
|
+
# Fortunately, ActorProxy defines the #_send_ method, which we can use to
|
141
|
+
# define a singleton_method on the actual proxied instance.
|
142
|
+
# We cannot just redefine the class method here, because it will break other
|
143
|
+
# tests.
|
144
|
+
|
145
|
+
start_filter_chain_mock_method = lambda do |message|
|
146
|
+
@method_called ||= 0
|
147
|
+
@method_called += 1
|
148
|
+
end
|
149
|
+
|
150
|
+
method_called_mock_method = lambda do
|
151
|
+
@method_called ||= 0
|
152
|
+
end
|
153
|
+
|
154
|
+
acceptor._send_(:define_singleton_method, :start_filter_chain, start_filter_chain_mock_method)
|
155
|
+
acceptor._send_(:define_singleton_method, :method_called, method_called_mock_method)
|
156
|
+
|
157
|
+
acceptor.start
|
158
|
+
|
159
|
+
instance = Bunny.instance
|
160
|
+
queue = instance.queues[:some_queue]
|
161
|
+
queue.should_not be_nil
|
162
|
+
|
163
|
+
block = queue.block
|
164
|
+
block.call({})
|
165
|
+
|
166
|
+
acceptor.method_called.should == 1
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|