tochtli 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +7 -0
  2. data/.travis.yml +14 -0
  3. data/Gemfile +32 -0
  4. data/History.md +138 -0
  5. data/README.md +46 -0
  6. data/Rakefile +50 -0
  7. data/VERSION +1 -0
  8. data/assets/communication.png +0 -0
  9. data/assets/layers.png +0 -0
  10. data/examples/01-screencap-service/Gemfile +3 -0
  11. data/examples/01-screencap-service/README.md +5 -0
  12. data/examples/01-screencap-service/client.rb +15 -0
  13. data/examples/01-screencap-service/common.rb +15 -0
  14. data/examples/01-screencap-service/server.rb +26 -0
  15. data/examples/02-log-analyzer/Gemfile +3 -0
  16. data/examples/02-log-analyzer/README.md +5 -0
  17. data/examples/02-log-analyzer/client.rb +95 -0
  18. data/examples/02-log-analyzer/common.rb +33 -0
  19. data/examples/02-log-analyzer/sample.log +10001 -0
  20. data/examples/02-log-analyzer/server.rb +133 -0
  21. data/lib/tochtli.rb +177 -0
  22. data/lib/tochtli/active_record_connection_cleaner.rb +9 -0
  23. data/lib/tochtli/application.rb +135 -0
  24. data/lib/tochtli/base_client.rb +135 -0
  25. data/lib/tochtli/base_controller.rb +360 -0
  26. data/lib/tochtli/controller_manager.rb +99 -0
  27. data/lib/tochtli/engine.rb +15 -0
  28. data/lib/tochtli/message.rb +114 -0
  29. data/lib/tochtli/rabbit_client.rb +36 -0
  30. data/lib/tochtli/rabbit_connection.rb +249 -0
  31. data/lib/tochtli/reply_queue.rb +129 -0
  32. data/lib/tochtli/simple_validation.rb +23 -0
  33. data/lib/tochtli/test.rb +9 -0
  34. data/lib/tochtli/test/client.rb +28 -0
  35. data/lib/tochtli/test/controller.rb +66 -0
  36. data/lib/tochtli/test/integration.rb +78 -0
  37. data/lib/tochtli/test/memory_cache.rb +22 -0
  38. data/lib/tochtli/test/test_case.rb +191 -0
  39. data/lib/tochtli/test/test_unit.rb +22 -0
  40. data/lib/tochtli/version.rb +3 -0
  41. data/log_generator.rb +11 -0
  42. data/test/base_client_test.rb +68 -0
  43. data/test/controller_functional_test.rb +87 -0
  44. data/test/controller_integration_test.rb +274 -0
  45. data/test/controller_manager_test.rb +75 -0
  46. data/test/dummy/Rakefile +7 -0
  47. data/test/dummy/config/application.rb +36 -0
  48. data/test/dummy/config/boot.rb +4 -0
  49. data/test/dummy/config/database.yml +3 -0
  50. data/test/dummy/config/environment.rb +5 -0
  51. data/test/dummy/config/rabbit.yml +4 -0
  52. data/test/dummy/db/.gitkeep +0 -0
  53. data/test/dummy/log/.gitkeep +0 -0
  54. data/test/key_matcher_test.rb +100 -0
  55. data/test/log/.gitkeep +0 -0
  56. data/test/message_test.rb +80 -0
  57. data/test/rabbit_client_test.rb +71 -0
  58. data/test/rabbit_connection_test.rb +151 -0
  59. data/test/test_helper.rb +32 -0
  60. data/test/version_test.rb +8 -0
  61. data/tochtli.gemspec +129 -0
  62. metadata +259 -0
@@ -0,0 +1,129 @@
1
+ module Tochtli
2
+ class ReplyQueue
3
+ attr_reader :connection, :logger, :queue
4
+
5
+ def initialize(rabbit_connection, logger=nil)
6
+ @connection = rabbit_connection
7
+ @logger = logger || rabbit_connection.logger
8
+ @message_handlers = {}
9
+ @message_timeout_threads = {}
10
+
11
+ subscribe
12
+ end
13
+
14
+ def name
15
+ @queue.name
16
+ end
17
+
18
+ def subscribe
19
+ channel = @connection.channel
20
+ exchange = @connection.exchange
21
+
22
+ @queue = channel.queue('', exclusive: true, auto_delete: true)
23
+ @original_queue_name = @queue.name
24
+ @queue.bind exchange, routing_key: @queue.name
25
+
26
+ @consumer = Consumer.new(self, channel, @queue)
27
+ @consumer.on_delivery(&method(:on_delivery))
28
+
29
+ @queue.subscribe_with(@consumer)
30
+ end
31
+
32
+ def reconnect(channel)
33
+ if @queue
34
+ channel.connection.logger.debug "Recovering reply queue binding (original: #{@original_queue_name}, current: #{@queue.name})"
35
+
36
+ # Re-bind queue after name change (auto-generated new on server has been re-generated)
37
+ exchange = @connection.create_exchange(channel)
38
+ @queue.unbind exchange, routing_key: @original_queue_name
39
+ @queue.bind exchange, routing_key: @queue.name
40
+ end
41
+
42
+ @original_queue_name = @queue.name
43
+ end
44
+
45
+ def register_message_handler(message, handler=nil, timeout=nil, &block)
46
+ @message_handlers[message.id] = handler || block
47
+ if timeout
48
+ timeout_thread = Thread.start do
49
+ sleep timeout
50
+ logger.warn "[#{Time.now} AMQP] TIMEOUT on message '#{message.id}' timeout: #{timeout}"
51
+ handle_timeout message
52
+ end
53
+ @message_timeout_threads[message.id] = timeout_thread
54
+ end
55
+ end
56
+
57
+ def on_delivery(delivery_info, metadata, payload)
58
+ class_name = metadata.type.camelize.gsub(/[^a-zA-Z0-9\:]/, '_') # basic sanity
59
+ reply_class = eval(class_name)
60
+ reply = reply_class.new({}, metadata)
61
+ attributes = JSON.parse(payload)
62
+ reply.attributes = attributes
63
+
64
+ logger.debug "[#{Time.now} AMQP] Replay for #{reply.properties.correlation_id}: #{reply.inspect}"
65
+
66
+ handle_reply reply
67
+
68
+ rescue Exception
69
+ logger.error $!
70
+ logger.error $!.backtrace.join("\n")
71
+ end
72
+
73
+ def handle_reply(reply, correlation_id=nil)
74
+ correlation_id ||= reply.properties.correlation_id if reply.is_a?(Tochtli::Message)
75
+ raise ArgumentError, "Correlated message ID expected" unless correlation_id
76
+ if (handler = @message_handlers.delete(correlation_id))
77
+ if (timeout_thread = @message_timeout_threads.delete(correlation_id))
78
+ timeout_thread.kill
79
+ timeout_thread.join # make sure timeout thread is dead
80
+ end
81
+
82
+ if !reply.is_a?(Tochtli::ErrorMessage) && !reply.is_a?(Exception)
83
+
84
+ begin
85
+
86
+ handler.call(reply)
87
+
88
+ rescue Exception
89
+ logger.error $!
90
+ logger.error $!.backtrace.join("\n")
91
+ handler.on_error($!)
92
+ end
93
+
94
+ else
95
+ handler.on_error(reply)
96
+ end
97
+
98
+ else
99
+ logger.error "[Tochtli::ReplyQueue] Unexpected message delivery '#{correlation_id}':\n\t#{reply.inspect})"
100
+ end
101
+ end
102
+
103
+ def handle_timeout(original_message)
104
+ if (handler = @message_handlers.delete(original_message.id))
105
+ @message_timeout_threads.delete(original_message.id)
106
+ handler.on_timeout original_message
107
+ else
108
+ raise "Internal error, timeout handler not found for message: #{original_message.id}, #{original_message.inspect}"
109
+ end
110
+ end
111
+
112
+ class Consumer < ::Bunny::Consumer
113
+ def initialize(reply_queue, *args)
114
+ super(*args)
115
+ @reply_queue = reply_queue
116
+ end
117
+
118
+ def recover_from_network_failure
119
+ super
120
+ @reply_queue.reconnect(@channel)
121
+ rescue Exception
122
+ logger = channel.connection.logger
123
+ logger.error $!
124
+ logger.error $!.backtrace.join("\n")
125
+ raise
126
+ end
127
+ end
128
+ end
129
+ end
@@ -0,0 +1,23 @@
1
+ module Tochtli
2
+ module SimpleValidation
3
+ attr_reader :errors
4
+
5
+ def add_error(message)
6
+ @errors << message
7
+ end
8
+
9
+ def valid?
10
+ @errors = []
11
+ validate
12
+ !@errors || @errors.empty?
13
+ end
14
+
15
+ def invalid?
16
+ !valid?
17
+ end
18
+
19
+ # abstract method
20
+ def validate
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,9 @@
1
+ module Tochtli
2
+ module Test
3
+ autoload :Helpers, 'tochtli/test/test_case'
4
+ autoload :Client, 'tochtli/test/client'
5
+ autoload :Controller, 'tochtli/test/controller'
6
+ autoload :Integration, 'tochtli/test/integration'
7
+ autoload :MemoryCache, 'tochtli/test/memory_cache'
8
+ end
9
+ end
@@ -0,0 +1,28 @@
1
+ require_relative 'test_case'
2
+
3
+ module Tochtli
4
+ module Test
5
+ module Client
6
+ extend UnitTestSupport if defined?(::Test::Unit)
7
+ include Tochtli::Test::Helpers
8
+
9
+ def before_setup
10
+ super
11
+ @logger = Tochtli.logger
12
+ @client = Tochtli::RabbitClient.new(@connection, @logger)
13
+ @reply_queue = @client.reply_queue
14
+ end
15
+
16
+ def create_reply(reply_class, original_message, attributes)
17
+ properties = TestMessageProperties.new(nil, reply_class.generate_id, original_message.id)
18
+ reply_class.new(attributes, properties)
19
+ end
20
+
21
+ def handle_reply(reply_class, original_message, attributes)
22
+ reply = create_reply(reply_class, original_message, attributes)
23
+ @reply_queue.handle_reply reply
24
+ reply
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,66 @@
1
+ require_relative 'test_case'
2
+
3
+ module Tochtli
4
+ module Test
5
+ module Controller
6
+ module ControllerClassSupport
7
+ def included(base)
8
+ super
9
+ base.class_eval do
10
+ extend Uber::InheritableAttr
11
+ inheritable_attr :controller_class
12
+
13
+ def self.tests(controller_class)
14
+ self.controller_class = controller_class
15
+ end
16
+ end
17
+ end
18
+ end
19
+
20
+ extend UnitTestSupport if defined?(::Test::Unit)
21
+ extend ControllerClassSupport
22
+ include Tochtli::Test::Helpers
23
+
24
+ class RoutingNotFound < StandardError
25
+ end
26
+
27
+
28
+ def before_setup
29
+ super
30
+ @cache = Object.const_defined?(:ActiveSupport) ? ActiveSupport::Cache::MemoryStore.new : Tochtli::Test::MemoryCache.new
31
+ @logger = Tochtli.logger
32
+ self.class.controller_class.setup(@connection, @cache, @logger)
33
+ @dispatcher = self.class.controller_class.dispatcher
34
+ @message_index = 0
35
+ end
36
+
37
+ def after_teardown
38
+ self.class.controller_class.stop
39
+ super
40
+ end
41
+
42
+ def publish(message)
43
+ @message_index += 1
44
+ delivery_info = TestDeliveryInfo.new(message.routing_key)
45
+ properties = TestMessageProperties.new("test.reply", @message_index)
46
+ payload = message.to_json
47
+
48
+ @message, @reply = nil
49
+
50
+ unless @dispatcher.process_message(delivery_info, properties, payload, {})
51
+ if (reply = @connection.publications.first) && reply[:message].is_a?(Tochtli::ErrorMessage)
52
+ raise "Process error: #{reply[:message].message}"
53
+ else
54
+ raise RoutingNotFound, "Message #{message.class.name} not processed by #{self.class.controller_class} - #{message.inspect}."
55
+ end
56
+ end
57
+
58
+ reply = @connection.publications.first
59
+ if reply && reply[:routing_key] == "test.reply" && reply[:correlation_id] == @message_index
60
+ @connection.publications.shift
61
+ @reply = reply[:message]
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,78 @@
1
+ require_relative 'test_case'
2
+
3
+ module Tochtli
4
+ # Ensure all queues are temporary
5
+ BaseController.queue_durable = false
6
+ BaseController.queue_auto_delete = true
7
+
8
+ module Test
9
+ module Integration
10
+ extend UnitTestSupport if defined?(::Test::Unit)
11
+
12
+ def before_setup
13
+ super
14
+ @logger = Tochtli.logger
15
+ @logger.level = Logger::DEBUG
16
+ @client = Tochtli::RabbitClient.new(nil, @logger)
17
+ @connection = @client.rabbit_connection
18
+ @controller_manager = Tochtli::ControllerManager.instance
19
+ @controller_manager.setup(connection: @connection, logger: @logger)
20
+ @controller_manager.start(:all)
21
+
22
+ # Reply support
23
+ @mutex = Mutex.new
24
+ @cv = ConditionVariable.new
25
+ end
26
+
27
+ def after_teardown
28
+ begin
29
+ @controller_manager.stop if @controller_manager
30
+ rescue Timeout::Error
31
+ warn "Unable to stop controller manager: #{$!} [#{$!.class}]"
32
+ end
33
+ super
34
+ end
35
+
36
+ private
37
+
38
+ def publish(message, options={})
39
+ @reply = nil
40
+ timeout = options.fetch(:timeout, 1.0)
41
+ @reply_message_class = options[:expect]
42
+ @reply_handler = options[:reply_handler]
43
+
44
+ if @reply_message_class || @reply_handler
45
+ handler = @reply_handler || method(:synchronous_reply_handler)
46
+ if handler.is_a?(Proc)
47
+ @client.reply_queue.register_message_handler message, &handler
48
+ else
49
+ @client.reply_queue.register_message_handler message, handler, timeout
50
+ end
51
+ end
52
+
53
+ @client.publish message
54
+
55
+ if @reply_message_class && !@reply_handler
56
+ synchronous_timeout_handler(message, timeout)
57
+ end
58
+ end
59
+
60
+ def synchronous_reply_handler(reply)
61
+ assert_kind_of @reply_message_class, reply, "Unexpected reply"
62
+ @mutex.synchronize do
63
+ @reply = reply
64
+ @cv.signal
65
+ end
66
+ end
67
+
68
+ def synchronous_timeout_handler(message, timeout)
69
+ @mutex.synchronize { @cv.wait(@mutex, timeout) unless @reply }
70
+
71
+ raise "Reply on #{message.class.name} timeout" unless @reply
72
+ raise @reply.message if @reply.is_a?(Tochtli::ErrorMessage)
73
+
74
+ @reply
75
+ end
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,22 @@
1
+ require 'mini_cache'
2
+
3
+ module Tochtli
4
+ module Test
5
+ # a simple proxy to replicate ActiveSupport cache interface using mini store
6
+ class MemoryCache
7
+ attr_reader :store
8
+
9
+ def initialize
10
+ @store = MiniCache::Store.new
11
+ end
12
+
13
+ def write(name, value)
14
+ store.set(name, value)
15
+ end
16
+
17
+ def read(name)
18
+ store.get(name)
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,191 @@
1
+ require_relative 'test_unit' if defined?(::Test::Unit)
2
+
3
+ module Tochtli
4
+ module Test
5
+ module Helpers
6
+ extend UnitTestSupport if defined?(::Test::Unit)
7
+
8
+ def before_setup
9
+ super
10
+ @connection = TestRabbitConnection.new
11
+ end
12
+
13
+ def assert_published(message_class, attributes={})
14
+ publication = @connection.get_publication
15
+ assert !publication.nil?, "No message published"
16
+ @message = publication[:message]
17
+ assert_kind_of message_class, @message
18
+ attributes.each do |attr_name, value|
19
+ assert_equal value, @message.send(attr_name), "Message attribute :#{attr_name} value does not match"
20
+ end
21
+ yield @message if block_given?
22
+ @message
23
+ end
24
+
25
+ def expect_published(message_class, attributes={})
26
+ @connection.callback do
27
+ assert_published message_class, attributes
28
+ yield @message
29
+ end
30
+ end
31
+ end
32
+
33
+ class TestRabbitConnection
34
+ attr_reader :channel, :exchange, :publications
35
+
36
+ def initialize
37
+ @channel = TestRabbitChannel.new(self)
38
+ @exchange = TestRabbitExchange.new
39
+ @publications = []
40
+ @queues = {}
41
+ end
42
+
43
+ def exchange_name
44
+ @exchange.name
45
+ end
46
+
47
+ def reply_queue
48
+ @reply_queue ||= Tochtli::ReplyQueue.new(self)
49
+ end
50
+
51
+ def publish(routing_key, message, options={})
52
+ @publications << options.merge(routing_key: routing_key, message: message)
53
+ run_callback
54
+ end
55
+
56
+ def get_publication
57
+ @publications.shift
58
+ end
59
+
60
+ def callback(&block)
61
+ @callback = block
62
+ end
63
+
64
+ def run_callback
65
+ if @callback
66
+ @callback.call
67
+ @callback = nil
68
+ end
69
+ end
70
+
71
+ def logger
72
+ Logger.new(STDOUT)
73
+ end
74
+
75
+ def queue(name=nil, routing_keys=[], options={})
76
+ queue = @queues[name]
77
+ unless queue
78
+ @queues[name] = queue = TestQueue.new(@channel, name, options)
79
+ end
80
+ queue
81
+ end
82
+
83
+ def queue_exists?(name)
84
+ @queues.has_key?(name)
85
+ end
86
+
87
+ def create_channel(consumer_pool_size = 1)
88
+ TestRabbitChannel.new(self)
89
+ end
90
+ end
91
+
92
+ class TestRabbitChannel
93
+ def initialize(connection)
94
+ @connection = connection
95
+ end
96
+
97
+ def queue(name, options={})
98
+ @connection.queue(name, [], options)
99
+ end
100
+
101
+ [:topic, :fanout, :direct].each do |type|
102
+ define_method type do |name, options|
103
+ TestRabbitExchange.new(name, options)
104
+ end
105
+ end
106
+
107
+ def generate_consumer_tag
108
+ "test-consumer-tag-#{rand(1000)}"
109
+ end
110
+ end
111
+
112
+ class TestRabbitExchange
113
+ attr_reader :name
114
+
115
+ def initialize(name='test.exchange', options={})
116
+ @name = name
117
+ end
118
+ end
119
+
120
+ class TestQueue
121
+ attr_reader :channel, :name, :options, :routing_key
122
+
123
+ def initialize(channel, name, options)
124
+ @name = name
125
+ @channel = channel
126
+ @options = options
127
+ end
128
+
129
+ def bind(exchange, options)
130
+ @routing_key = options[:routing_key]
131
+ end
132
+
133
+ def subscribe(*args)
134
+ TestConsumer.new
135
+ end
136
+
137
+ def subscribe_with(*args)
138
+ TestConsumer.new
139
+ end
140
+ end
141
+
142
+ class TestConsumer
143
+ def cancel
144
+ TestConsumer.new
145
+ end
146
+ end
147
+
148
+ class TestDeliveryInfo
149
+ attr_reader :routing_key, :exchange
150
+
151
+ def initialize(routing_key, exchange='TestExchange')
152
+ @routing_key = routing_key
153
+ @exchange = exchange
154
+ end
155
+
156
+ def [](key)
157
+ send(key)
158
+ end
159
+ end
160
+
161
+ class TestMessageProperties
162
+ attr_reader :reply_to, :message_id, :correlation_id
163
+
164
+ def initialize(reply_to, message_id=nil, correlation_id=nil)
165
+ @reply_to = reply_to
166
+ @message_id = message_id
167
+ @correlation_id = correlation_id
168
+ end
169
+
170
+ def [](key)
171
+ send(key)
172
+ end
173
+ end
174
+
175
+ class TestMessageHandler
176
+ attr_reader :reply, :timeout_message, :error
177
+
178
+ def call(reply)
179
+ @reply = reply
180
+ end
181
+
182
+ def on_timeout(original_message=nil)
183
+ @timeout_message = original_message
184
+ end
185
+
186
+ def on_error(error)
187
+ @error = error
188
+ end
189
+ end
190
+ end
191
+ end