batsir 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. data/.document +5 -0
  2. data/.rspec +3 -0
  3. data/.travis.yml +10 -0
  4. data/Gemfile +22 -0
  5. data/LICENSE.txt +20 -0
  6. data/README.md +79 -0
  7. data/Rakefile +55 -0
  8. data/VERSION +1 -0
  9. data/batsir.gemspec +104 -0
  10. data/batsir.png +0 -0
  11. data/lib/batsir.rb +73 -0
  12. data/lib/batsir/acceptors/acceptor.rb +45 -0
  13. data/lib/batsir/acceptors/amqp_acceptor.rb +19 -0
  14. data/lib/batsir/amqp.rb +45 -0
  15. data/lib/batsir/chain.rb +33 -0
  16. data/lib/batsir/config.rb +34 -0
  17. data/lib/batsir/dsl/dsl_mappings.rb +96 -0
  18. data/lib/batsir/filter.rb +12 -0
  19. data/lib/batsir/filter_queue.rb +30 -0
  20. data/lib/batsir/logo.rb +36 -0
  21. data/lib/batsir/notifiers/amqp_notifier.rb +16 -0
  22. data/lib/batsir/notifiers/notifier.rb +39 -0
  23. data/lib/batsir/registry.rb +15 -0
  24. data/lib/batsir/stage.rb +94 -0
  25. data/lib/batsir/stage_worker.rb +86 -0
  26. data/lib/batsir/transformers/field_transformer.rb +40 -0
  27. data/lib/batsir/transformers/json_input_transformer.rb +9 -0
  28. data/lib/batsir/transformers/json_output_transformer.rb +9 -0
  29. data/lib/batsir/transformers/transformer.rb +15 -0
  30. data/spec/batsir/acceptors/acceptor_spec.rb +136 -0
  31. data/spec/batsir/acceptors/amqp_acceptor_spec.rb +169 -0
  32. data/spec/batsir/chain_spec.rb +31 -0
  33. data/spec/batsir/dsl/chain_mapping_spec.rb +117 -0
  34. data/spec/batsir/dsl/stage_mapping_spec.rb +435 -0
  35. data/spec/batsir/filter_queue_spec.rb +74 -0
  36. data/spec/batsir/filter_spec.rb +12 -0
  37. data/spec/batsir/notifiers/amqp_notifier_spec.rb +117 -0
  38. data/spec/batsir/notifiers/notifier_spec.rb +73 -0
  39. data/spec/batsir/stage_spec.rb +678 -0
  40. data/spec/batsir/stage_worker_spec.rb +128 -0
  41. data/spec/batsir/support/bunny_mocks.rb +62 -0
  42. data/spec/batsir/support/mock_filters.rb +43 -0
  43. data/spec/batsir/transformers/field_transformer_spec.rb +73 -0
  44. data/spec/batsir/transformers/json_input_transformer_spec.rb +22 -0
  45. data/spec/batsir/transformers/json_output_transformer_spec.rb +18 -0
  46. data/spec/batsir/transformers/transformer_spec.rb +22 -0
  47. data/spec/spec_helper.rb +22 -0
  48. metadata +220 -0
@@ -0,0 +1,128 @@
1
+ require File.join( File.dirname(__FILE__), "..", "spec_helper" )
2
+
3
+ describe Batsir::StageWorker do
4
+ context "with respect to including the StageWorker module" do
5
+ before :all do
6
+ class TestWorker
7
+ def self.stage_name
8
+ @stage_name_called = true
9
+ "TestWorker"
10
+ end
11
+
12
+ def self.stage_name_called
13
+ @stage_name_called || false
14
+ end
15
+
16
+ def self.initialize_filter_queue
17
+ @initialization_count ||= 0
18
+ @initialization_count += 1
19
+ end
20
+
21
+ def self.initialization_count
22
+ @initialization_count
23
+ end
24
+
25
+ include Batsir::StageWorker
26
+ end
27
+ end
28
+
29
+ it "should register workers once they include the stage worker" do
30
+ Batsir::Registry.get("TestWorker").should == TestWorker
31
+ end
32
+
33
+ it "should call the queue initialization method" do
34
+ TestWorker.initialization_count.should == 1
35
+ end
36
+
37
+ it "should call the stage name class method to register itself under" do
38
+ TestWorker.stage_name_called.should be_true
39
+ end
40
+ end
41
+
42
+ it "should be possible to set an filter queue" do
43
+ Batsir::StageWorker.instance_methods.map{|m| m.to_s}.should include "filter_queue="
44
+ end
45
+
46
+ context "With respect to executing" do
47
+ before :all do
48
+ class TestWorker
49
+ def self.stage_name
50
+ "TestWorker"
51
+ end
52
+
53
+ def self.intitialize_filter_queue
54
+ @initialization_count ||= 0
55
+ @initialization_count += 1
56
+ end
57
+
58
+ include Batsir::StageWorker
59
+ end
60
+
61
+ class TestNotifier
62
+ def notify(message)
63
+ @notify_count ||= 0
64
+ @notify_count += 1
65
+ end
66
+
67
+ def notify_count
68
+ @notify_count ||= 0
69
+ end
70
+ end
71
+ end
72
+
73
+ before :each do
74
+ chain = Batsir::Chain.new
75
+
76
+ stage_options = {
77
+ :chain => chain,
78
+ }
79
+ stage = Batsir::Stage.new(stage_options)
80
+
81
+ stage.add_filter SumFilter
82
+ stage.add_filter AverageFilter
83
+ stage.add_notifier( :notification_queue_1, {:queue => :somequeue} )
84
+ stage.add_notifier( :notification_queue_2 )
85
+ end
86
+
87
+ it "should not execute when no operation queue is set" do
88
+ stage_worker = TestWorker.new
89
+ stage_worker.execute({}).should be_false
90
+ end
91
+
92
+ it "should execute all operations in the operation queue when an #execute message is received" do
93
+ filter_queue = Batsir::FilterQueue.new
94
+ filter_queue.add SumFilter.new
95
+ filter_queue.add AverageFilter.new
96
+
97
+ stage_worker = TestWorker.new
98
+ stage_worker.filter_queue = filter_queue
99
+
100
+ queue = stage_worker.filter_queue
101
+ queue.each do |filter|
102
+ filter.execute_count.should == 0
103
+ end
104
+
105
+ stage_worker.execute({}).should be_true
106
+
107
+ queue.each do |filter|
108
+ filter.execute_count.should == 1
109
+ end
110
+ end
111
+
112
+ it "should call #notify on all notifiers in the filter queue when an #execute message is received" do
113
+ filter_queue = Batsir::FilterQueue.new
114
+ filter_queue.add_notifier TestNotifier.new
115
+
116
+ stage_worker = TestWorker.new
117
+ stage_worker.filter_queue = filter_queue
118
+
119
+ notifier = filter_queue.notifiers.first
120
+ notifier.should_not be_nil
121
+ notifier.notify_count.should == 0
122
+
123
+ stage_worker.execute({}).should be_true
124
+
125
+ notifier.notify_count.should == 1
126
+ end
127
+ end
128
+ end
@@ -0,0 +1,62 @@
1
+ module Bunny
2
+ def self.instance
3
+ @instance
4
+ end
5
+
6
+ def self.run(options = {})
7
+ @instance = BunnyInstance.new(options)
8
+ yield @instance
9
+ end
10
+
11
+ class BunnyInstance
12
+ attr_accessor :queues
13
+ attr_accessor :options
14
+ attr_accessor :exchange
15
+
16
+ def initialize(options = {})
17
+ @options = options
18
+ @queues = {}
19
+ end
20
+
21
+ def exchange(exchange = nil)
22
+ @exchange = BunnyExchange.new(exchange) if exchange
23
+ @exchange
24
+ end
25
+
26
+ def queue(queue)
27
+ @queues[queue] = BunnyQueue.new
28
+ end
29
+ end
30
+
31
+ class BunnyExchange
32
+ attr_accessor :name
33
+ attr_accessor :key
34
+ attr_accessor :message
35
+
36
+ def initialize(name)
37
+ @name = name
38
+ end
39
+
40
+ def publish(message, options = {})
41
+ @key = options[:key]
42
+ @message = message
43
+ end
44
+ end
45
+
46
+ class BunnyQueue
47
+ attr_accessor :arguments
48
+ attr_accessor :block
49
+ attr_accessor :bound_exchange
50
+ attr_accessor :bound_key
51
+
52
+ def bind(exchange, options)
53
+ @bound_exchange = exchange
54
+ @bound_key = options[:key]
55
+ end
56
+
57
+ def subscribe(*args, &block)
58
+ @arguments = *args
59
+ @block = block
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,43 @@
1
+ module Batsir::MockBehavior
2
+ attr_accessor :execute_count
3
+ def initialize(options = {})
4
+ super
5
+ @execute_count = 0
6
+ end
7
+
8
+ def execute(message)
9
+ @execute_count += 1
10
+ end
11
+ end
12
+
13
+ class Batsir::MockFilter < Batsir::Filter
14
+ include Batsir::MockBehavior
15
+ end
16
+
17
+ class PersistenceFilter < Batsir::MockFilter
18
+ end
19
+
20
+ class SumFilter < Batsir::MockFilter
21
+ end
22
+
23
+ class AverageFilter < Batsir::MockFilter
24
+ end
25
+
26
+ class Batsir::RetrievalFilter
27
+ include Batsir::MockBehavior
28
+ end
29
+
30
+ class Batsir::MessagePrinter < Batsir::Filter
31
+ def execute(message)
32
+ puts "In #{self.class.to_s}#execute(#{message.inspect})"
33
+ puts message
34
+ message
35
+ end
36
+ end
37
+
38
+ class Batsir::MessageCreator < Batsir::Filter
39
+ def execute(message)
40
+ puts "In #{self.class.to_s}#execute(#{message.inspect})"
41
+ {:id => message}
42
+ end
43
+ end
@@ -0,0 +1,73 @@
1
+ require File.join( File.dirname(__FILE__), "..", "..", "spec_helper" )
2
+
3
+ describe Batsir::Transformers::FieldTransformer do
4
+ let( :transformer_class ) do
5
+ Batsir::Transformers::FieldTransformer
6
+ end
7
+
8
+ it "should accept an options hash in its initializer" do
9
+ transformer_instance = transformer_class.new( {} )
10
+ transformer_instance.should_not be_nil
11
+ transformer_instance.should be_a transformer_class
12
+ end
13
+
14
+ it "should be possible to set a field mapping using the 'fields' option" do
15
+ field_mapping = {:foo => :bar}
16
+ transformer = transformer_class.new( :fields => field_mapping )
17
+ transformer.fields.should == field_mapping
18
+ end
19
+
20
+ it "should use the fields mapping given in an options hash to transform the message using #transform" do
21
+ field_mapping = {:foo => :bar}
22
+ transformer = transformer_class.new( :fields => field_mapping )
23
+
24
+ message = {:bar => "bar"}
25
+ transformed_message = transformer.transform(message)
26
+ transformed_message.should have_key :foo
27
+ transformed_message.should_not have_key :bar
28
+ transformed_message[:foo].should == "bar"
29
+ end
30
+
31
+ it "should be possible to use symbols and string based keys and values all the same" do
32
+ field_mapping = {:foo => "bar", "john" => :doe}
33
+ transformer = transformer_class.new( :fields => field_mapping )
34
+
35
+ message = {:bar => "foo", "doe" => :john}
36
+ transformed_message = transformer.transform(message)
37
+ transformed_message[:foo].should == "foo"
38
+ transformed_message[:john].should == :john
39
+ end
40
+
41
+ it "should remove options not in the fields option when a fields option is given" do
42
+ field_mapping = {:foo => :bar}
43
+ transformer = transformer_class.new( :fields => field_mapping )
44
+
45
+ message = {:bar => "bar", :john => :doe}
46
+ transformed_message = transformer.transform(message)
47
+ transformed_message.should have_key :foo
48
+ transformed_message.should_not have_key :bar
49
+ transformed_message.should_not have_key :john
50
+ end
51
+
52
+ it "should not remove fields when no mapping is given" do
53
+ transformer = transformer_class.new
54
+
55
+ message = {:bar => "bar", :john => :doe}
56
+ transformed_message = transformer.transform(message)
57
+ transformed_message.should have_key :bar
58
+ transformed_message.should have_key :john
59
+ end
60
+
61
+ it "should correctly handle more complex field mapping" do
62
+ field_mapping = {:id => :old_id}
63
+ transformer = transformer_class.new( :fields => field_mapping )
64
+
65
+ message = {:id => 2, :old_id => 1, :john => :doe}
66
+ transformed_message = transformer.transform(message)
67
+
68
+ transformed_message.should have_key :id
69
+ transformed_message.should_not have_key :old_id
70
+ transformed_message.should_not have_key :john
71
+ end
72
+
73
+ end
@@ -0,0 +1,22 @@
1
+ require File.join( File.dirname(__FILE__), "..", "..", "spec_helper" )
2
+
3
+ describe Batsir::Transformers::JSONInputTransformer do
4
+ it "should be a Batsir::Transformers::Transformer" do
5
+ subject.should be_a Batsir::Transformers::Transformer
6
+ end
7
+
8
+ it "should transform the input to a ruby object" do
9
+ some_json = '{"foo" : "bar"}'
10
+ result = subject.transform(some_json)
11
+ result.should be_a Hash
12
+ end
13
+
14
+ it "should transform the input using string names" do
15
+ some_json = '{"foo" : "bar"}'
16
+ result = subject.transform(some_json)
17
+
18
+ result[:foo].should be_nil
19
+ result["foo"].should_not be_nil
20
+ result["foo"].should == "bar"
21
+ end
22
+ end
@@ -0,0 +1,18 @@
1
+ require File.join( File.dirname(__FILE__), "..", "..", "spec_helper" )
2
+
3
+ describe Batsir::Transformers::JSONOutputTransformer do
4
+ it "should be a Batsir::Transformers::Transformer" do
5
+ subject.should be_a Batsir::Transformers::Transformer
6
+ end
7
+
8
+ it "should transform a hash to a valid json hash" do
9
+ some_hash = {:foo => :bar}
10
+ some_hash.should be_a Hash
11
+
12
+ result = subject.transform(some_hash)
13
+ result.should be_a String
14
+
15
+ result2 = JSON.parse(result)
16
+ result2.should be_a Hash
17
+ end
18
+ end
@@ -0,0 +1,22 @@
1
+ require File.join( File.dirname(__FILE__), "..", "..", "spec_helper" )
2
+
3
+ describe Batsir::Transformers::Transformer do
4
+ let( :transformer_class ) do
5
+ Batsir::Transformers::Transformer
6
+ end
7
+
8
+ it "should accept an options hash in its initializer" do
9
+ transformer_instance = transformer_class.new( {} )
10
+ transformer_instance.should_not be_nil
11
+ transformer_instance.should be_a transformer_class
12
+ end
13
+
14
+ it "should have a #transform method" do
15
+ transformer_class.instance_methods.map{|m| m.to_s}.should include "transform"
16
+ end
17
+
18
+ it "should return the message by default" do
19
+ message = {:foo => :bar}
20
+ transformer_class.new.transform(message).should == message
21
+ end
22
+ end
@@ -0,0 +1,22 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+ require 'rspec/core'
11
+
12
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
14
+ require 'batsir'
15
+ require 'batsir/support/mock_filters'
16
+ require 'batsir/support/bunny_mocks'
17
+ Celluloid.logger.level = Logger::ERROR
18
+ RSpec.configure do |config|
19
+ config.after(:each) do
20
+ Celluloid.shutdown
21
+ end
22
+ end
metadata ADDED
@@ -0,0 +1,220 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: batsir
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - J.W. Koelewijn
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2012-06-28 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: bundler
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>'
20
+ - !ruby/object:Gem::Version
21
+ version: 1.0.0
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ! '>'
28
+ - !ruby/object:Gem::Version
29
+ version: 1.0.0
30
+ - !ruby/object:Gem::Dependency
31
+ name: jeweler
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: '0'
38
+ type: :runtime
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '0'
46
+ - !ruby/object:Gem::Dependency
47
+ name: blockenspiel
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ~>
52
+ - !ruby/object:Gem::Version
53
+ version: 0.4.3
54
+ type: :runtime
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ~>
60
+ - !ruby/object:Gem::Version
61
+ version: 0.4.3
62
+ - !ruby/object:Gem::Dependency
63
+ name: celluloid
64
+ requirement: !ruby/object:Gem::Requirement
65
+ none: false
66
+ requirements:
67
+ - - ! '>='
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ type: :runtime
71
+ prerelease: false
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ! '>='
76
+ - !ruby/object:Gem::Version
77
+ version: '0'
78
+ - !ruby/object:Gem::Dependency
79
+ name: sidekiq
80
+ requirement: !ruby/object:Gem::Requirement
81
+ none: false
82
+ requirements:
83
+ - - ! '>='
84
+ - !ruby/object:Gem::Version
85
+ version: '0'
86
+ type: :runtime
87
+ prerelease: false
88
+ version_requirements: !ruby/object:Gem::Requirement
89
+ none: false
90
+ requirements:
91
+ - - ! '>='
92
+ - !ruby/object:Gem::Version
93
+ version: '0'
94
+ - !ruby/object:Gem::Dependency
95
+ name: bunny
96
+ requirement: !ruby/object:Gem::Requirement
97
+ none: false
98
+ requirements:
99
+ - - ! '>='
100
+ - !ruby/object:Gem::Version
101
+ version: '0'
102
+ type: :runtime
103
+ prerelease: false
104
+ version_requirements: !ruby/object:Gem::Requirement
105
+ none: false
106
+ requirements:
107
+ - - ! '>='
108
+ - !ruby/object:Gem::Version
109
+ version: '0'
110
+ - !ruby/object:Gem::Dependency
111
+ name: json
112
+ requirement: !ruby/object:Gem::Requirement
113
+ none: false
114
+ requirements:
115
+ - - ! '>='
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ none: false
122
+ requirements:
123
+ - - ! '>='
124
+ - !ruby/object:Gem::Version
125
+ version: '0'
126
+ description: ! "Batsir uses so called stages to define operation queues. These operation
127
+ queus\n consist of several operations that will be executed one after the other.
128
+ Each stage\n is defined by its name and the queue on which it will listen. Once
129
+ a message is received\n on the queue, it is dispatched to a worker in a seperate
130
+ thread that will pass the message\n to each operation in the operation queue.\n
131
+ Operation queues can have 4 different operations, 1 common operation type, and 3
132
+ special \n purpose operations: retrieval operations (which are always executed before
133
+ all other operations),\n persistence operations (which are always executed after
134
+ the common operations, but before the\n notification operations) and notification
135
+ operations (which will always be executed last)\n This makes it possible to create
136
+ chains of stages to perform tasks that depend on each\n other, but otherwise have
137
+ a low coupling"
138
+ email: jwkoelewijn@gmail.com
139
+ executables: []
140
+ extensions: []
141
+ extra_rdoc_files:
142
+ - LICENSE.txt
143
+ - README.md
144
+ files:
145
+ - .document
146
+ - .rspec
147
+ - .travis.yml
148
+ - Gemfile
149
+ - LICENSE.txt
150
+ - README.md
151
+ - Rakefile
152
+ - VERSION
153
+ - batsir.gemspec
154
+ - batsir.png
155
+ - lib/batsir.rb
156
+ - lib/batsir/acceptors/acceptor.rb
157
+ - lib/batsir/acceptors/amqp_acceptor.rb
158
+ - lib/batsir/amqp.rb
159
+ - lib/batsir/chain.rb
160
+ - lib/batsir/config.rb
161
+ - lib/batsir/dsl/dsl_mappings.rb
162
+ - lib/batsir/filter.rb
163
+ - lib/batsir/filter_queue.rb
164
+ - lib/batsir/logo.rb
165
+ - lib/batsir/notifiers/amqp_notifier.rb
166
+ - lib/batsir/notifiers/notifier.rb
167
+ - lib/batsir/registry.rb
168
+ - lib/batsir/stage.rb
169
+ - lib/batsir/stage_worker.rb
170
+ - lib/batsir/transformers/field_transformer.rb
171
+ - lib/batsir/transformers/json_input_transformer.rb
172
+ - lib/batsir/transformers/json_output_transformer.rb
173
+ - lib/batsir/transformers/transformer.rb
174
+ - spec/batsir/acceptors/acceptor_spec.rb
175
+ - spec/batsir/acceptors/amqp_acceptor_spec.rb
176
+ - spec/batsir/chain_spec.rb
177
+ - spec/batsir/dsl/chain_mapping_spec.rb
178
+ - spec/batsir/dsl/stage_mapping_spec.rb
179
+ - spec/batsir/filter_queue_spec.rb
180
+ - spec/batsir/filter_spec.rb
181
+ - spec/batsir/notifiers/amqp_notifier_spec.rb
182
+ - spec/batsir/notifiers/notifier_spec.rb
183
+ - spec/batsir/stage_spec.rb
184
+ - spec/batsir/stage_worker_spec.rb
185
+ - spec/batsir/support/bunny_mocks.rb
186
+ - spec/batsir/support/mock_filters.rb
187
+ - spec/batsir/transformers/field_transformer_spec.rb
188
+ - spec/batsir/transformers/json_input_transformer_spec.rb
189
+ - spec/batsir/transformers/json_output_transformer_spec.rb
190
+ - spec/batsir/transformers/transformer_spec.rb
191
+ - spec/spec_helper.rb
192
+ homepage: http://github.com/jwkoelewijn/batsir
193
+ licenses:
194
+ - MIT
195
+ post_install_message:
196
+ rdoc_options: []
197
+ require_paths:
198
+ - lib
199
+ required_ruby_version: !ruby/object:Gem::Requirement
200
+ none: false
201
+ requirements:
202
+ - - ! '>='
203
+ - !ruby/object:Gem::Version
204
+ version: '0'
205
+ segments:
206
+ - 0
207
+ hash: -1524934688976294279
208
+ required_rubygems_version: !ruby/object:Gem::Requirement
209
+ none: false
210
+ requirements:
211
+ - - ! '>='
212
+ - !ruby/object:Gem::Version
213
+ version: '0'
214
+ requirements: []
215
+ rubyforge_project:
216
+ rubygems_version: 1.8.22
217
+ signing_key:
218
+ specification_version: 3
219
+ summary: Batsir is an execution platform for stage based operation queue execution
220
+ test_files: []