batsir 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. data/.document +5 -0
  2. data/.rspec +3 -0
  3. data/.travis.yml +10 -0
  4. data/Gemfile +22 -0
  5. data/LICENSE.txt +20 -0
  6. data/README.md +79 -0
  7. data/Rakefile +55 -0
  8. data/VERSION +1 -0
  9. data/batsir.gemspec +104 -0
  10. data/batsir.png +0 -0
  11. data/lib/batsir.rb +73 -0
  12. data/lib/batsir/acceptors/acceptor.rb +45 -0
  13. data/lib/batsir/acceptors/amqp_acceptor.rb +19 -0
  14. data/lib/batsir/amqp.rb +45 -0
  15. data/lib/batsir/chain.rb +33 -0
  16. data/lib/batsir/config.rb +34 -0
  17. data/lib/batsir/dsl/dsl_mappings.rb +96 -0
  18. data/lib/batsir/filter.rb +12 -0
  19. data/lib/batsir/filter_queue.rb +30 -0
  20. data/lib/batsir/logo.rb +36 -0
  21. data/lib/batsir/notifiers/amqp_notifier.rb +16 -0
  22. data/lib/batsir/notifiers/notifier.rb +39 -0
  23. data/lib/batsir/registry.rb +15 -0
  24. data/lib/batsir/stage.rb +94 -0
  25. data/lib/batsir/stage_worker.rb +86 -0
  26. data/lib/batsir/transformers/field_transformer.rb +40 -0
  27. data/lib/batsir/transformers/json_input_transformer.rb +9 -0
  28. data/lib/batsir/transformers/json_output_transformer.rb +9 -0
  29. data/lib/batsir/transformers/transformer.rb +15 -0
  30. data/spec/batsir/acceptors/acceptor_spec.rb +136 -0
  31. data/spec/batsir/acceptors/amqp_acceptor_spec.rb +169 -0
  32. data/spec/batsir/chain_spec.rb +31 -0
  33. data/spec/batsir/dsl/chain_mapping_spec.rb +117 -0
  34. data/spec/batsir/dsl/stage_mapping_spec.rb +435 -0
  35. data/spec/batsir/filter_queue_spec.rb +74 -0
  36. data/spec/batsir/filter_spec.rb +12 -0
  37. data/spec/batsir/notifiers/amqp_notifier_spec.rb +117 -0
  38. data/spec/batsir/notifiers/notifier_spec.rb +73 -0
  39. data/spec/batsir/stage_spec.rb +678 -0
  40. data/spec/batsir/stage_worker_spec.rb +128 -0
  41. data/spec/batsir/support/bunny_mocks.rb +62 -0
  42. data/spec/batsir/support/mock_filters.rb +43 -0
  43. data/spec/batsir/transformers/field_transformer_spec.rb +73 -0
  44. data/spec/batsir/transformers/json_input_transformer_spec.rb +22 -0
  45. data/spec/batsir/transformers/json_output_transformer_spec.rb +18 -0
  46. data/spec/batsir/transformers/transformer_spec.rb +22 -0
  47. data/spec/spec_helper.rb +22 -0
  48. metadata +220 -0
@@ -0,0 +1,86 @@
1
+ require 'sidekiq'
2
+
3
+ module Batsir
4
+ module StageWorker
5
+ attr_accessor :filter_queue
6
+
7
+ def self.included(base)
8
+ Registry.register(base.stage_name, base)
9
+ base.initialize_filter_queue
10
+ end
11
+
12
+ def perform(message)
13
+ execute(message)
14
+ end
15
+
16
+ def execute(message)
17
+ return false unless @filter_queue
18
+ @filter_queue.filters.each do |filter|
19
+ message = filter.execute(message)
20
+ return false if message.nil?
21
+ end
22
+ @filter_queue.notifiers.each do |notifier|
23
+ notifier.notify(message)
24
+ end
25
+ true
26
+ end
27
+
28
+ def self.compile_from(stage)
29
+ code = <<-EOF
30
+ class #{stage.name.capitalize.gsub(' ','')}Worker
31
+ def self.stage_name
32
+ "#{stage.name}"
33
+ end
34
+
35
+ def initialize
36
+ @filter_queue = self.class.filter_queue
37
+ end
38
+
39
+ def self.filter_queue
40
+ @filter_queue
41
+ end
42
+
43
+ def self.initialize_filter_queue
44
+ @filter_queue = Batsir::FilterQueue.new
45
+ EOF
46
+
47
+ stage.filter_declarations.each do |filter_declaration|
48
+ code << <<-EOF
49
+ @filter_queue.add #{filter_declaration.filter.to_s}.new(#{filter_declaration.options.to_s})
50
+ EOF
51
+ end
52
+
53
+ stage.notifiers.each do |notifier, options_set|
54
+ options_set.each do |options|
55
+ code << <<-EOF
56
+ notifier = #{notifier.to_s}.new(#{options.to_s})
57
+ EOF
58
+
59
+ if stage.notifier_transformers.any?
60
+ stage.notifier_transformers.each do |transformer_declaration|
61
+ code << <<-EOF
62
+ notifier.add_transformer #{transformer_declaration.transformer}.new(#{transformer_declaration.options.to_s})
63
+ EOF
64
+ end
65
+ else
66
+ code << <<-EOF
67
+ notifier.add_transformer Batsir::Transformers::JSONOutputTransformer.new
68
+ EOF
69
+ end
70
+ code << <<-EOF
71
+ @filter_queue.add_notifier notifier
72
+ EOF
73
+ end
74
+ end
75
+
76
+ code << <<-EOF
77
+ end
78
+
79
+ include Sidekiq::Worker
80
+ include Batsir::StageWorker
81
+ end
82
+ EOF
83
+ code
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,40 @@
1
+ module Batsir
2
+ module Transformers
3
+ class FieldTransformer < Transformer
4
+ attr_accessor :fields
5
+
6
+ def fields
7
+ @fields ||= {}
8
+ end
9
+
10
+ def fields=(hash)
11
+ @fields = {}
12
+ hash.each do |k, v|
13
+ @fields[k.to_sym] = v.to_sym
14
+ end
15
+ end
16
+
17
+ def transform(message)
18
+ fields = self.fields
19
+ if fields.any? && message.respond_to?(:keys)
20
+ symbolized_message_keys = {}
21
+ message.keys.each do |key|
22
+ symbolized_message_keys[key.to_sym] = key
23
+ end
24
+
25
+ fields_to_remove = symbolized_message_keys.keys - fields.keys - fields.values
26
+
27
+ fields.each do |new, old|
28
+ message[new] = message.delete(symbolized_message_keys[old])
29
+ end
30
+
31
+ fields_to_remove.each do |field|
32
+ message.delete(symbolized_message_keys[field])
33
+ end
34
+ end
35
+ message
36
+ end
37
+
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,9 @@
1
+ module Batsir
2
+ module Transformers
3
+ class JSONInputTransformer < Transformer
4
+ def transform(message)
5
+ JSON.parse(message, :symbolize_names => false)
6
+ end
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,9 @@
1
+ module Batsir
2
+ module Transformers
3
+ class JSONOutputTransformer < Transformer
4
+ def transform(message)
5
+ JSON.dump(message)
6
+ end
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,15 @@
1
+ module Batsir
2
+ module Transformers
3
+ class Transformer
4
+ def initialize(options = {})
5
+ options.each do |attr, value|
6
+ self.send("#{attr}=", value)
7
+ end
8
+ end
9
+
10
+ def transform(message)
11
+ message
12
+ end
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,136 @@
1
+ require File.join( File.dirname(__FILE__), "..", "..", "spec_helper" )
2
+
3
+ describe Batsir::Acceptors::Acceptor do
4
+ let( :acceptor_class ) {
5
+ Batsir::Acceptors::Acceptor
6
+ }
7
+
8
+ it "should be a Celluloid actor" do
9
+ acceptor_class.ancestors.should include Celluloid
10
+ end
11
+
12
+ it "should be possible to set the stage name for the acceptor" do
13
+ acceptor = acceptor_class.new
14
+ stage_name = "some stage"
15
+ acceptor.stage_name = stage_name
16
+ acceptor.stage_name.should == stage_name
17
+ end
18
+
19
+ it "should have a #start method" do
20
+ acceptor_class.instance_methods.map{|im| im.to_s}.should include "start"
21
+ end
22
+
23
+ it "should have an #start_filter_chain method" do
24
+ acceptor_class.instance_methods.map{|im| im.to_s}.should include "start_filter_chain"
25
+ end
26
+
27
+ it "should be possible to initialize an Acceptor with on options hash" do
28
+ acceptor = acceptor_class.new({})
29
+ acceptor.should_not be_nil
30
+ end
31
+
32
+ it "should look up a worker class when the #start_filter_chain method is called" do
33
+ acceptor = acceptor_class.new
34
+ stage_name = "some stage"
35
+ acceptor.stage_name = stage_name
36
+ Batsir::Registry.should_receive(:get).with(stage_name)
37
+ acceptor.start_filter_chain({})
38
+ end
39
+
40
+ it "should have a transformer_queue" do
41
+ acceptor = acceptor_class.new
42
+ acceptor.transformer_queue.should_not be_nil
43
+ end
44
+
45
+ it "should initially have an empty transformer_queue" do
46
+ acceptor = acceptor_class.new
47
+ acceptor.transformer_queue.should_not be_nil
48
+ acceptor.transformer_queue.should be_empty
49
+ end
50
+
51
+ it "should be possible to add a transformer to the transformer_queue" do
52
+ transformer = :transformer
53
+
54
+ acceptor = acceptor_class.new
55
+ acceptor.add_transformer transformer
56
+
57
+ acceptor.transformer_queue.should_not be_empty
58
+ acceptor.transformer_queue.size.should == 1
59
+ acceptor.transformer_queue.first.should == :transformer
60
+ end
61
+
62
+ it "should be possible to add a transformer multiple times" do
63
+ transformer = :transformer
64
+
65
+ acceptor = acceptor_class.new
66
+ acceptor.add_transformer transformer
67
+ acceptor.add_transformer transformer
68
+
69
+ acceptor.transformer_queue.should_not be_empty
70
+ acceptor.transformer_queue.size.should == 2
71
+ end
72
+
73
+ it "should be possible to set a cancellator" do
74
+ cancellator = :cancel
75
+ acceptor = acceptor_class.new
76
+ acceptor.cancellator = cancellator
77
+ acceptor.cancellator.should == cancellator
78
+ end
79
+
80
+ it "should be possible to set a cancellator in the constructor using a hash" do
81
+ cancellator = :cancel
82
+ acceptor = acceptor_class.new(:cancellator => cancellator)
83
+ acceptor.cancellator.should == cancellator
84
+ end
85
+
86
+ it "should call the #perform_async on the worker class when #start_filter_chain is called" do
87
+ class MockWorker
88
+ def self.stage_name
89
+ "mock_stage"
90
+ end
91
+
92
+ def self.initialize_filter_queue
93
+ end
94
+
95
+ def self.perform_async(*args)
96
+ end
97
+
98
+ include Batsir::StageWorker
99
+ end
100
+
101
+ acceptor = acceptor_class.new
102
+ stage_name = "some stage"
103
+
104
+ acceptor.stage_name = stage_name
105
+
106
+ Batsir::Registry.register(stage_name, MockWorker)
107
+ MockWorker.should_receive(:perform_async)
108
+ acceptor.start_filter_chain({})
109
+ end
110
+
111
+ it "should call #transform on the acceptor transformers" do
112
+ class MockTransformer < Batsir::Transformers::Transformer
113
+ def transform(message)
114
+ @@transformed ||= 0
115
+ @@transformed += 1
116
+ end
117
+
118
+ def self.transformed
119
+ @@transformed ||= 0
120
+ @@transformed
121
+ end
122
+ end
123
+
124
+ acceptor = acceptor_class.new
125
+ stage_name = "some stage"
126
+
127
+ acceptor.stage_name = stage_name
128
+ acceptor.add_transformer MockTransformer.new
129
+
130
+ MockTransformer.transformed.should == 0
131
+
132
+ acceptor.start_filter_chain({})
133
+
134
+ MockTransformer.transformed.should == 1
135
+ end
136
+ end
@@ -0,0 +1,169 @@
1
+ require File.join( File.dirname(__FILE__), "..", "..", "spec_helper" )
2
+
3
+ describe Batsir::Acceptors::AMQPAcceptor do
4
+
5
+ let(:acceptor_class){
6
+ Batsir::Acceptors::AMQPAcceptor
7
+ }
8
+
9
+ context "With respect to setting options" do
10
+
11
+ it "should be a Batsir::Acceptors::Acceptor" do
12
+ acceptor_class.ancestors.should include Batsir::Acceptors::Acceptor
13
+ end
14
+
15
+ it "should be possible to set the queue on which to listen" do
16
+ acceptor = acceptor_class.new(:queue => :queue)
17
+ acceptor.queue.should == :queue
18
+ end
19
+
20
+ it "should be possible to set the host of the amqp broker" do
21
+ acceptor = acceptor_class.new(:host => 'localhost')
22
+ acceptor.host.should == 'localhost'
23
+ end
24
+
25
+ it "should be possible to set the port of the amqp broker" do
26
+ acceptor = acceptor_class.new(:port => 1234)
27
+ acceptor.port.should == 1234
28
+ end
29
+
30
+ it "should be possible to set the username with which to connect to the broker" do
31
+ acceptor = acceptor_class.new(:username => 'some_user')
32
+ acceptor.username.should == 'some_user'
33
+ end
34
+
35
+ it "should be possible to set the password with which to connect to the broker" do
36
+ acceptor = acceptor_class.new(:password => 'password')
37
+ acceptor.password.should == 'password'
38
+ end
39
+
40
+ it "should be possible to set the vhost to use on the broker" do
41
+ acceptor = acceptor_class.new(:vhost => '/vhost')
42
+ acceptor.vhost.should == '/vhost'
43
+ end
44
+
45
+ it "should be possible to set the exchange to use on the broker" do
46
+ acceptor = acceptor_class.new(:exchange => 'amq.direct')
47
+ acceptor.exchange.should == 'amq.direct'
48
+ end
49
+
50
+ it "should default to amqp://guest:guest@localhost:5672/ with direct exchange on vhost ''" do
51
+ acceptor = acceptor_class.new(:queue => :somequeue)
52
+ acceptor.queue.should == :somequeue
53
+ acceptor.host.should == 'localhost'
54
+ acceptor.port.should == 5672
55
+ acceptor.username.should == 'guest'
56
+ acceptor.password.should == 'guest'
57
+ acceptor.vhost.should == '/'
58
+ acceptor.exchange.should == 'amq.direct'
59
+ end
60
+ end
61
+
62
+ context "with respect to starting the acceptor" do
63
+ def new_acceptor(options = {})
64
+ acceptor_class.new(options)
65
+ end
66
+
67
+ it "should connect to the configured host" do
68
+ acceptor = new_acceptor(:host => 'somehost')
69
+ acceptor.start
70
+ instance = Bunny.instance
71
+ instance.options[:host].should == 'somehost'
72
+ end
73
+
74
+ it "should connect to the configured port" do
75
+ acceptor = new_acceptor(:port => 1234)
76
+ acceptor.start
77
+ instance = Bunny.instance
78
+ instance.options[:port].should == 1234
79
+ end
80
+
81
+ it "should connect with the configured username" do
82
+ acceptor = new_acceptor(:username => 'user')
83
+ acceptor.start
84
+ instance = Bunny.instance
85
+ instance.options[:user].should == 'user'
86
+ end
87
+
88
+ it "should connect with the configured password" do
89
+ acceptor = new_acceptor(:password => 'pass')
90
+ acceptor.start
91
+ instance = Bunny.instance
92
+ instance.options[:pass].should == 'pass'
93
+ end
94
+
95
+ it "should connect to the configured vhost" do
96
+ acceptor = new_acceptor(:vhost => '/vhost')
97
+ acceptor.start
98
+ instance = Bunny.instance
99
+ instance.options[:vhost].should == '/vhost'
100
+ end
101
+
102
+ it "should declare the configured exchange" do
103
+ acceptor = new_acceptor(:exchange => 'some_exchange')
104
+ acceptor.start
105
+ instance = Bunny.instance
106
+ instance.exchange.name.should == 'some_exchange'
107
+ end
108
+
109
+ it "should bind the configured exchange to the queue" do
110
+ acceptor = new_acceptor(:exchange => 'some_exchange', :queue => :queue)
111
+ acceptor.start
112
+ instance = Bunny.instance
113
+ queue = instance.queues[:queue]
114
+ queue.bound_exchange.name.should == 'some_exchange'
115
+ queue.bound_key.should == :queue
116
+ end
117
+
118
+ it "should start listening on the configured queue" do
119
+ acceptor = new_acceptor(:queue => :some_queue)
120
+ acceptor.start
121
+ instance = Bunny.instance
122
+ instance.queues.size.should == 1
123
+ instance.queues.keys.should include :some_queue
124
+ end
125
+
126
+ it "should initialize the subscription with the acceptor's cancellator" do
127
+ cancellator = :cancellator
128
+ acceptor = new_acceptor(:queue => :some_queue, :cancellator => cancellator)
129
+ acceptor.start
130
+ instance = Bunny.instance
131
+ puts instance.queues[:some_queue].arguments.first[:cancellator].should == cancellator
132
+ end
133
+
134
+ it "should call the #start_filter_chain method when a message is received" do
135
+ acceptor = new_acceptor(:queue => :some_queue)
136
+
137
+ # Because acceptor is a Celluloid Actor, it is not possible to define a method
138
+ # on the instance directly, because you actually hold a reference to a
139
+ # ActorProxy.
140
+ # Fortunately, ActorProxy defines the #_send_ method, which we can use to
141
+ # define a singleton_method on the actual proxied instance.
142
+ # We cannot just redefine the class method here, because it will break other
143
+ # tests.
144
+
145
+ start_filter_chain_mock_method = lambda do |message|
146
+ @method_called ||= 0
147
+ @method_called += 1
148
+ end
149
+
150
+ method_called_mock_method = lambda do
151
+ @method_called ||= 0
152
+ end
153
+
154
+ acceptor._send_(:define_singleton_method, :start_filter_chain, start_filter_chain_mock_method)
155
+ acceptor._send_(:define_singleton_method, :method_called, method_called_mock_method)
156
+
157
+ acceptor.start
158
+
159
+ instance = Bunny.instance
160
+ queue = instance.queues[:some_queue]
161
+ queue.should_not be_nil
162
+
163
+ block = queue.block
164
+ block.call({})
165
+
166
+ acceptor.method_called.should == 1
167
+ end
168
+ end
169
+ end