tochtli 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +7 -0
  2. data/.travis.yml +14 -0
  3. data/Gemfile +32 -0
  4. data/History.md +138 -0
  5. data/README.md +46 -0
  6. data/Rakefile +50 -0
  7. data/VERSION +1 -0
  8. data/assets/communication.png +0 -0
  9. data/assets/layers.png +0 -0
  10. data/examples/01-screencap-service/Gemfile +3 -0
  11. data/examples/01-screencap-service/README.md +5 -0
  12. data/examples/01-screencap-service/client.rb +15 -0
  13. data/examples/01-screencap-service/common.rb +15 -0
  14. data/examples/01-screencap-service/server.rb +26 -0
  15. data/examples/02-log-analyzer/Gemfile +3 -0
  16. data/examples/02-log-analyzer/README.md +5 -0
  17. data/examples/02-log-analyzer/client.rb +95 -0
  18. data/examples/02-log-analyzer/common.rb +33 -0
  19. data/examples/02-log-analyzer/sample.log +10001 -0
  20. data/examples/02-log-analyzer/server.rb +133 -0
  21. data/lib/tochtli.rb +177 -0
  22. data/lib/tochtli/active_record_connection_cleaner.rb +9 -0
  23. data/lib/tochtli/application.rb +135 -0
  24. data/lib/tochtli/base_client.rb +135 -0
  25. data/lib/tochtli/base_controller.rb +360 -0
  26. data/lib/tochtli/controller_manager.rb +99 -0
  27. data/lib/tochtli/engine.rb +15 -0
  28. data/lib/tochtli/message.rb +114 -0
  29. data/lib/tochtli/rabbit_client.rb +36 -0
  30. data/lib/tochtli/rabbit_connection.rb +249 -0
  31. data/lib/tochtli/reply_queue.rb +129 -0
  32. data/lib/tochtli/simple_validation.rb +23 -0
  33. data/lib/tochtli/test.rb +9 -0
  34. data/lib/tochtli/test/client.rb +28 -0
  35. data/lib/tochtli/test/controller.rb +66 -0
  36. data/lib/tochtli/test/integration.rb +78 -0
  37. data/lib/tochtli/test/memory_cache.rb +22 -0
  38. data/lib/tochtli/test/test_case.rb +191 -0
  39. data/lib/tochtli/test/test_unit.rb +22 -0
  40. data/lib/tochtli/version.rb +3 -0
  41. data/log_generator.rb +11 -0
  42. data/test/base_client_test.rb +68 -0
  43. data/test/controller_functional_test.rb +87 -0
  44. data/test/controller_integration_test.rb +274 -0
  45. data/test/controller_manager_test.rb +75 -0
  46. data/test/dummy/Rakefile +7 -0
  47. data/test/dummy/config/application.rb +36 -0
  48. data/test/dummy/config/boot.rb +4 -0
  49. data/test/dummy/config/database.yml +3 -0
  50. data/test/dummy/config/environment.rb +5 -0
  51. data/test/dummy/config/rabbit.yml +4 -0
  52. data/test/dummy/db/.gitkeep +0 -0
  53. data/test/dummy/log/.gitkeep +0 -0
  54. data/test/key_matcher_test.rb +100 -0
  55. data/test/log/.gitkeep +0 -0
  56. data/test/message_test.rb +80 -0
  57. data/test/rabbit_client_test.rb +71 -0
  58. data/test/rabbit_connection_test.rb +151 -0
  59. data/test/test_helper.rb +32 -0
  60. data/test/version_test.rb +8 -0
  61. data/tochtli.gemspec +129 -0
  62. metadata +259 -0
@@ -0,0 +1,135 @@
1
+ module Tochtli
2
+ class BaseClient
3
+ attr_reader :rabbit_client, :rabbit_connection, :logger
4
+
5
+ def initialize(rabbit_client_or_connection = nil, logger = nil)
6
+
7
+ logger ||= Tochtli.logger
8
+ case rabbit_client_or_connection
9
+ when Tochtli::RabbitClient
10
+ @rabbit_client = rabbit_client_or_connection
11
+ when Tochtli::RabbitConnection
12
+ @rabbit_client = Tochtli::RabbitClient.new(rabbit_client_or_connection, logger)
13
+ when NilClass
14
+ @rabbit_client = Tochtli::RabbitClient.new(nil, logger)
15
+ else
16
+ raise ArgumentError, "Tochtli::RabbitClient or Tochtli::RabbitConnection expected, got: #{rabbit_client_or_connection.class}"
17
+ end
18
+ @rabbit_connection = @rabbit_client.rabbit_connection
19
+ @logger = logger
20
+ end
21
+
22
+ def publish(*args)
23
+ rabbit_client.publish(*args)
24
+ end
25
+
26
+ def wait_for_confirms
27
+ rabbit_client.wait_for_confirms
28
+ end
29
+
30
+ class InternalServiceError < StandardError
31
+ attr_reader :service_error
32
+
33
+ def initialize(service_error, message)
34
+ @service_error = service_error
35
+ super message
36
+ end
37
+ end
38
+
39
+ class AbstractMessageHandler
40
+ def reconstruct_exception(error_message)
41
+ if error_message.is_a?(Exception)
42
+ error_message
43
+ else
44
+ if error_message.is_a?(ErrorMessage)
45
+ error = error_message.error
46
+ message = error_message.message
47
+ # Look for exception definition on client system
48
+ exception_class = error.constantize rescue nil
49
+ if exception_class < StandardError
50
+ # Use known exception if found
51
+ exception = exception_class.new(message)
52
+ else
53
+ # If the exception is not defined use generic InternalServiceError
54
+ exception = InternalServiceError.new(error, message)
55
+ end
56
+ else
57
+ exception = InternalServiceError.new(error_message.to_s)
58
+ end
59
+ exception.set_backtrace caller(0)
60
+ exception
61
+ end
62
+ end
63
+ end
64
+
65
+ class MessageHandler < AbstractMessageHandler
66
+ def initialize(external_handler)
67
+ @external_handler = external_handler
68
+ end
69
+
70
+ def on_timeout(original_message)
71
+ @external_handler.on_timeout
72
+ end
73
+
74
+ def on_error(error_message)
75
+ error = reconstruct_exception(error_message)
76
+ @external_handler.on_error error
77
+ end
78
+ end
79
+
80
+ class SyncMessageHandler < AbstractMessageHandler
81
+ include MonitorMixin
82
+ attr_reader :reply, :error
83
+
84
+ def initialize
85
+ super # initialize monitor
86
+ @cv = new_cond
87
+ end
88
+
89
+ def wait(timeout)
90
+ synchronize do
91
+ @cv.wait(timeout) unless handled?
92
+ end
93
+ on_timeout unless handled?
94
+ end
95
+
96
+ def wait!(timeout)
97
+ wait(timeout)
98
+ raise_error unless @reply
99
+ @reply
100
+ end
101
+
102
+ def handled?
103
+ @reply || @error
104
+ end
105
+
106
+ def on_timeout(original_message=nil)
107
+ synchronize do
108
+ @error = Timeout::Error.new(original_message ? "Unable to send message: #{original_message.inspect}" : "Service is not responding")
109
+ @cv.signal
110
+ end
111
+ end
112
+
113
+ def on_error(error_message)
114
+ synchronize do
115
+ @error = reconstruct_exception(error_message)
116
+ @cv.signal
117
+ end
118
+ end
119
+
120
+ def call(reply)
121
+ synchronize do
122
+ @reply = reply
123
+ @cv.signal
124
+ end
125
+ end
126
+
127
+ def raise_error
128
+ error = self.error || InternalServiceError.new("Unknwon", "Unknown Error")
129
+ raise error
130
+ end
131
+ end
132
+
133
+ end
134
+ end
135
+
@@ -0,0 +1,360 @@
1
+ require 'hooks'
2
+
3
+ module Tochtli
4
+ class BaseController
5
+ extend Uber::InheritableAttribute
6
+ include Hooks
7
+ include Hooks::InstanceHooks
8
+
9
+ inheritable_attr :routing_keys
10
+ inheritable_attr :message_handlers
11
+ inheritable_attr :work_pool_size
12
+
13
+ self.work_pool_size = 1 # default work pool size per controller instance
14
+
15
+ attr_reader :logger, :env, :message, :delivery_info
16
+
17
+ # Each controller can overwrite the queue name (default: controller.name.underscore)
18
+ inheritable_attr :queue_name
19
+
20
+ # Custom options for controller queue and exchange
21
+ inheritable_attr :queue_durable
22
+ self.queue_durable = true
23
+
24
+ inheritable_attr :queue_auto_delete
25
+ self.queue_auto_delete = false
26
+
27
+ inheritable_attr :queue_exclusive
28
+ self.queue_exclusive = false
29
+
30
+ inheritable_attr :exchange_type
31
+ self.exchange_type = :topic
32
+
33
+ inheritable_attr :exchange_name # read from configuration by default
34
+
35
+ inheritable_attr :exchange_durable
36
+ self.exchange_durable = true
37
+
38
+ # Message dispatcher created on start
39
+ inheritable_attr :dispatcher
40
+
41
+ define_hooks :before_setup, :after_setup,
42
+ :before_start, :after_start,
43
+ :before_stop, :after_stop,
44
+ :before_restart, :after_restart
45
+
46
+ class << self
47
+ def inherited(controller)
48
+ controller.routing_keys = Set.new
49
+ controller.message_handlers = Array.new
50
+ controller.queue_name = controller.name.underscore.gsub('::', '/')
51
+ ControllerManager.register(controller)
52
+ end
53
+
54
+ def bind(*routing_keys)
55
+ self.routing_keys.merge(routing_keys)
56
+ end
57
+
58
+ def on(message_class, method_name=nil, opts={}, &block)
59
+ if method_name.is_a?(Hash)
60
+ opts = method_name
61
+ method_name = nil
62
+ end
63
+ method = method_name ? method_name : block
64
+ raise ArgumentError, "Method name or block must be given" unless method
65
+
66
+ raise ArgumentError, "Message class expected, got: #{message_class}" unless message_class < Tochtli::Message
67
+
68
+ routing_key = opts[:routing_key] || message_class.routing_key
69
+ raise "Topic not set for message: #{message_class}" unless routing_key
70
+
71
+ self.message_handlers << MessageRoute.new(message_class, method, routing_key)
72
+ end
73
+
74
+ def off(routing_key)
75
+ self.message_handlers.delete_if {|route| route.routing_key == routing_key }
76
+ end
77
+
78
+
79
+ def setup(rabbit_connection, cache=nil, logger=nil)
80
+ run_hook :before_setup, rabbit_connection
81
+ self.dispatcher = Dispatcher.new(self, rabbit_connection, cache, logger || Tochtli.logger)
82
+ run_hook :after_setup, rabbit_connection
83
+ end
84
+
85
+ def start(queue_name=nil, routing_keys=nil, initial_env={})
86
+ run_hook :before_start, queue_name, initial_env
87
+ self.dispatcher.start(queue_name || self.queue_name, routing_keys || self.routing_keys, initial_env)
88
+ run_hook :after_start, queue_name, initial_env
89
+ end
90
+
91
+ def set_up?
92
+ !!self.dispatcher
93
+ end
94
+
95
+ def started?(queue_name=nil)
96
+ self.dispatcher && self.dispatcher.started?(queue_name)
97
+ end
98
+
99
+ def stop(options={})
100
+ if started?
101
+ queues = self.dispatcher.queues
102
+ run_hook :before_stop, queues
103
+
104
+ self.dispatcher.shutdown(options)
105
+ self.dispatcher = nil
106
+
107
+ run_hook :after_stop, queues
108
+ end
109
+ end
110
+
111
+ def restart(options={})
112
+ if started?
113
+ queues = self.dispatcher.queues
114
+ run_hook :before_restart, queues
115
+ self.dispatcher.restart options
116
+ run_hook :after_restart, queues
117
+ end
118
+ end
119
+
120
+ def find_message_route(routing_key)
121
+ raise "Routing not set up" if self.message_handlers.empty?
122
+ self.message_handlers.find {|handler| handler.pattern =~ routing_key }
123
+ end
124
+
125
+ def create_queue(rabbit_connection, queue_name=nil, routing_keys=nil)
126
+ queue_name = self.queue_name unless queue_name
127
+ routing_keys = self.routing_keys unless routing_keys
128
+ channel = rabbit_connection.create_channel(self.work_pool_size)
129
+ exchange_name = self.exchange_name || rabbit_connection.exchange_name
130
+ exchange = channel.send(self.exchange_type, exchange_name, durable: self.exchange_durable)
131
+ queue = channel.queue(queue_name,
132
+ durable: self.queue_durable,
133
+ exclusive: self.queue_exclusive,
134
+ auto_delete: self.queue_auto_delete)
135
+
136
+ routing_keys.each do |routing_key|
137
+ queue.bind(exchange, routing_key: routing_key)
138
+ end
139
+
140
+ queue
141
+ end
142
+ end
143
+
144
+ public
145
+
146
+ def initialize(rabbit_connection, cache, logger)
147
+ @rabbit_connection = rabbit_connection
148
+ @cache = cache
149
+ @logger = logger
150
+ end
151
+
152
+ def process_message(env)
153
+ @env = env
154
+ @action = env[:action]
155
+ @message = env[:message]
156
+ @delivery_info = env[:delivery_info]
157
+
158
+ if @action.is_a?(Proc)
159
+ instance_eval(&@action)
160
+ else
161
+ send @action
162
+ end
163
+
164
+ ensure
165
+ @env, @message, @delivery_info = nil
166
+ end
167
+
168
+ def reply(reply_message, reply_to=nil, message_id=nil)
169
+ if @message
170
+ reply_to ||= @message.properties.reply_to
171
+ message_id ||= @message.id
172
+ end
173
+
174
+ raise "The 'reply_to' queue name is not specified" unless reply_to
175
+
176
+ logger.debug "\tSending reply on #{message_id} to #{reply_to}: #{reply_message.inspect}."
177
+
178
+ @rabbit_connection.publish(reply_to,
179
+ reply_message,
180
+ correlation_id: message_id)
181
+ end
182
+
183
+ def rabbit_connection
184
+ self.class.dispatcher.rabbit_connection if self.class.set_up?
185
+ end
186
+
187
+ class MessageRoute < Struct.new(:message_class, :action, :routing_key, :pattern)
188
+ def initialize(message_class, action, routing_key)
189
+ super message_class, action, routing_key, KeyPattern.new(routing_key)
190
+ end
191
+ end
192
+
193
+ class Dispatcher
194
+ attr_reader :controller_class, :rabbit_connection, :cache, :logger
195
+
196
+ def initialize(controller_class, rabbit_connection, cache, logger)
197
+ @controller_class = controller_class
198
+ @rabbit_connection = rabbit_connection
199
+ @cache = cache
200
+ @logger = logger
201
+ @application = Tochtli.application.to_app
202
+ @queues = {}
203
+ @process_counter = ProcessCounter.new
204
+ @initial_env = nil
205
+ end
206
+
207
+ def start(queue_name, routing_keys, initial_env={})
208
+ subscribe_queue(queue_name, routing_keys, initial_env)
209
+ end
210
+
211
+ def restart(options={})
212
+ queues = @queues.dup
213
+
214
+ shutdown options
215
+
216
+ queues.each do |queue_name, queue_opts|
217
+ start queue_name, queue_opts[:initial_env]
218
+ end
219
+ end
220
+
221
+ def process_message(delivery_info, properties, payload, initial_env)
222
+ register_process_start
223
+
224
+ env = initial_env.merge(
225
+ delivery_info: delivery_info,
226
+ properties: properties,
227
+ payload: payload,
228
+ controller_class: controller_class,
229
+ rabbit_connection: rabbit_connection,
230
+ cache: cache,
231
+ logger: logger
232
+ )
233
+
234
+ @application.call(env)
235
+
236
+ rescue Exception => ex
237
+ logger.error "\nUNEXPECTED EXCEPTION: #{ex.class.name} (#{ex.message})"
238
+ logger.error ex.backtrace.join("\n")
239
+ false
240
+ ensure
241
+ register_process_end
242
+ end
243
+
244
+ def subscribe_queue(queue_name, routing_keys, initial_env={})
245
+ queue = controller_class.create_queue(@rabbit_connection, queue_name, routing_keys)
246
+ consumer = queue.subscribe do |delivery_info, metadata, payload|
247
+ process_message delivery_info, metadata, payload, initial_env
248
+ end
249
+
250
+ @queues[queue_name] = {
251
+ queue: queue,
252
+ consumer: consumer,
253
+ initial_env: initial_env
254
+ }
255
+ end
256
+
257
+ # Performs a graceful shutdown of dispatcher i.e. waits for all processes to end.
258
+ # If timeout is reached, forces the shutdown. Useful with dynamic reconfiguration of work pool size.
259
+ def shutdown(options={})
260
+ wait_for_processes options.fetch(:timeout, 15)
261
+ stop
262
+ end
263
+
264
+ def stop(queues=nil)
265
+ @queues.each_value { |queue_opts| queue_opts[:consumer].cancel }
266
+ rescue Bunny::ConnectionClosedError
267
+ # ignore closed connection error
268
+ ensure
269
+ @queues = {}
270
+ end
271
+
272
+ def started?(queue_name=nil)
273
+ if queue_name
274
+ @queues.has_key?(queue_name)
275
+ else
276
+ !@queues.empty?
277
+ end
278
+ end
279
+
280
+ def queues
281
+ @queues.map { |_, qh| qh[:queue] }
282
+ end
283
+
284
+ protected
285
+
286
+ def register_process_start
287
+ @process_counter.increment
288
+ end
289
+
290
+ def register_process_end
291
+ @process_counter.decrement
292
+ end
293
+
294
+ def wait_for_processes(timeout)
295
+ @process_counter.wait(timeout)
296
+ end
297
+
298
+ class ProcessCounter
299
+ include MonitorMixin
300
+
301
+ def initialize
302
+ super
303
+ @count = 0
304
+ @cv = new_cond
305
+ end
306
+
307
+ def increment
308
+ synchronize do
309
+ @count += 1
310
+ end
311
+ end
312
+
313
+ def decrement
314
+ synchronize do
315
+ @count -= 1 if @count > 0
316
+ @cv.signal if @count == 0
317
+ end
318
+ end
319
+
320
+ def wait(timeout)
321
+ synchronize do
322
+ @cv.wait(timeout) unless @count == 0
323
+ end
324
+ end
325
+ end
326
+ end
327
+
328
+ class KeyPattern
329
+ ASTERIX_EXP = '[a-zA-Z0-9]+'
330
+ HASH_EXP = '[a-zA-Z0-9\.]*'
331
+
332
+ def initialize(pattern)
333
+ @str = pattern
334
+ @simple = !pattern.include?('*') && !pattern.include?('#')
335
+ if @simple
336
+ @pattern = pattern
337
+ else
338
+ @pattern = Regexp.new('^' + pattern.gsub('.', '\\.').
339
+ gsub('*', ASTERIX_EXP).gsub(/(\\\.)?#(\\\.)?/, HASH_EXP) + '$')
340
+ end
341
+ end
342
+
343
+ def =~(key)
344
+ if @simple
345
+ @pattern == key
346
+ else
347
+ @pattern =~ key
348
+ end
349
+ end
350
+
351
+ def !~(key)
352
+ !(self =~ key)
353
+ end
354
+
355
+ def to_s
356
+ @str
357
+ end
358
+ end
359
+ end
360
+ end