tochtli 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.travis.yml +14 -0
- data/Gemfile +32 -0
- data/History.md +138 -0
- data/README.md +46 -0
- data/Rakefile +50 -0
- data/VERSION +1 -0
- data/assets/communication.png +0 -0
- data/assets/layers.png +0 -0
- data/examples/01-screencap-service/Gemfile +3 -0
- data/examples/01-screencap-service/README.md +5 -0
- data/examples/01-screencap-service/client.rb +15 -0
- data/examples/01-screencap-service/common.rb +15 -0
- data/examples/01-screencap-service/server.rb +26 -0
- data/examples/02-log-analyzer/Gemfile +3 -0
- data/examples/02-log-analyzer/README.md +5 -0
- data/examples/02-log-analyzer/client.rb +95 -0
- data/examples/02-log-analyzer/common.rb +33 -0
- data/examples/02-log-analyzer/sample.log +10001 -0
- data/examples/02-log-analyzer/server.rb +133 -0
- data/lib/tochtli.rb +177 -0
- data/lib/tochtli/active_record_connection_cleaner.rb +9 -0
- data/lib/tochtli/application.rb +135 -0
- data/lib/tochtli/base_client.rb +135 -0
- data/lib/tochtli/base_controller.rb +360 -0
- data/lib/tochtli/controller_manager.rb +99 -0
- data/lib/tochtli/engine.rb +15 -0
- data/lib/tochtli/message.rb +114 -0
- data/lib/tochtli/rabbit_client.rb +36 -0
- data/lib/tochtli/rabbit_connection.rb +249 -0
- data/lib/tochtli/reply_queue.rb +129 -0
- data/lib/tochtli/simple_validation.rb +23 -0
- data/lib/tochtli/test.rb +9 -0
- data/lib/tochtli/test/client.rb +28 -0
- data/lib/tochtli/test/controller.rb +66 -0
- data/lib/tochtli/test/integration.rb +78 -0
- data/lib/tochtli/test/memory_cache.rb +22 -0
- data/lib/tochtli/test/test_case.rb +191 -0
- data/lib/tochtli/test/test_unit.rb +22 -0
- data/lib/tochtli/version.rb +3 -0
- data/log_generator.rb +11 -0
- data/test/base_client_test.rb +68 -0
- data/test/controller_functional_test.rb +87 -0
- data/test/controller_integration_test.rb +274 -0
- data/test/controller_manager_test.rb +75 -0
- data/test/dummy/Rakefile +7 -0
- data/test/dummy/config/application.rb +36 -0
- data/test/dummy/config/boot.rb +4 -0
- data/test/dummy/config/database.yml +3 -0
- data/test/dummy/config/environment.rb +5 -0
- data/test/dummy/config/rabbit.yml +4 -0
- data/test/dummy/db/.gitkeep +0 -0
- data/test/dummy/log/.gitkeep +0 -0
- data/test/key_matcher_test.rb +100 -0
- data/test/log/.gitkeep +0 -0
- data/test/message_test.rb +80 -0
- data/test/rabbit_client_test.rb +71 -0
- data/test/rabbit_connection_test.rb +151 -0
- data/test/test_helper.rb +32 -0
- data/test/version_test.rb +8 -0
- data/tochtli.gemspec +129 -0
- metadata +259 -0
@@ -0,0 +1,135 @@
|
|
1
|
+
module Tochtli
|
2
|
+
class BaseClient
|
3
|
+
attr_reader :rabbit_client, :rabbit_connection, :logger
|
4
|
+
|
5
|
+
def initialize(rabbit_client_or_connection = nil, logger = nil)
|
6
|
+
|
7
|
+
logger ||= Tochtli.logger
|
8
|
+
case rabbit_client_or_connection
|
9
|
+
when Tochtli::RabbitClient
|
10
|
+
@rabbit_client = rabbit_client_or_connection
|
11
|
+
when Tochtli::RabbitConnection
|
12
|
+
@rabbit_client = Tochtli::RabbitClient.new(rabbit_client_or_connection, logger)
|
13
|
+
when NilClass
|
14
|
+
@rabbit_client = Tochtli::RabbitClient.new(nil, logger)
|
15
|
+
else
|
16
|
+
raise ArgumentError, "Tochtli::RabbitClient or Tochtli::RabbitConnection expected, got: #{rabbit_client_or_connection.class}"
|
17
|
+
end
|
18
|
+
@rabbit_connection = @rabbit_client.rabbit_connection
|
19
|
+
@logger = logger
|
20
|
+
end
|
21
|
+
|
22
|
+
def publish(*args)
|
23
|
+
rabbit_client.publish(*args)
|
24
|
+
end
|
25
|
+
|
26
|
+
def wait_for_confirms
|
27
|
+
rabbit_client.wait_for_confirms
|
28
|
+
end
|
29
|
+
|
30
|
+
class InternalServiceError < StandardError
|
31
|
+
attr_reader :service_error
|
32
|
+
|
33
|
+
def initialize(service_error, message)
|
34
|
+
@service_error = service_error
|
35
|
+
super message
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
class AbstractMessageHandler
|
40
|
+
def reconstruct_exception(error_message)
|
41
|
+
if error_message.is_a?(Exception)
|
42
|
+
error_message
|
43
|
+
else
|
44
|
+
if error_message.is_a?(ErrorMessage)
|
45
|
+
error = error_message.error
|
46
|
+
message = error_message.message
|
47
|
+
# Look for exception definition on client system
|
48
|
+
exception_class = error.constantize rescue nil
|
49
|
+
if exception_class < StandardError
|
50
|
+
# Use known exception if found
|
51
|
+
exception = exception_class.new(message)
|
52
|
+
else
|
53
|
+
# If the exception is not defined use generic InternalServiceError
|
54
|
+
exception = InternalServiceError.new(error, message)
|
55
|
+
end
|
56
|
+
else
|
57
|
+
exception = InternalServiceError.new(error_message.to_s)
|
58
|
+
end
|
59
|
+
exception.set_backtrace caller(0)
|
60
|
+
exception
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
class MessageHandler < AbstractMessageHandler
|
66
|
+
def initialize(external_handler)
|
67
|
+
@external_handler = external_handler
|
68
|
+
end
|
69
|
+
|
70
|
+
def on_timeout(original_message)
|
71
|
+
@external_handler.on_timeout
|
72
|
+
end
|
73
|
+
|
74
|
+
def on_error(error_message)
|
75
|
+
error = reconstruct_exception(error_message)
|
76
|
+
@external_handler.on_error error
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
class SyncMessageHandler < AbstractMessageHandler
|
81
|
+
include MonitorMixin
|
82
|
+
attr_reader :reply, :error
|
83
|
+
|
84
|
+
def initialize
|
85
|
+
super # initialize monitor
|
86
|
+
@cv = new_cond
|
87
|
+
end
|
88
|
+
|
89
|
+
def wait(timeout)
|
90
|
+
synchronize do
|
91
|
+
@cv.wait(timeout) unless handled?
|
92
|
+
end
|
93
|
+
on_timeout unless handled?
|
94
|
+
end
|
95
|
+
|
96
|
+
def wait!(timeout)
|
97
|
+
wait(timeout)
|
98
|
+
raise_error unless @reply
|
99
|
+
@reply
|
100
|
+
end
|
101
|
+
|
102
|
+
def handled?
|
103
|
+
@reply || @error
|
104
|
+
end
|
105
|
+
|
106
|
+
def on_timeout(original_message=nil)
|
107
|
+
synchronize do
|
108
|
+
@error = Timeout::Error.new(original_message ? "Unable to send message: #{original_message.inspect}" : "Service is not responding")
|
109
|
+
@cv.signal
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def on_error(error_message)
|
114
|
+
synchronize do
|
115
|
+
@error = reconstruct_exception(error_message)
|
116
|
+
@cv.signal
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def call(reply)
|
121
|
+
synchronize do
|
122
|
+
@reply = reply
|
123
|
+
@cv.signal
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def raise_error
|
128
|
+
error = self.error || InternalServiceError.new("Unknwon", "Unknown Error")
|
129
|
+
raise error
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
@@ -0,0 +1,360 @@
|
|
1
|
+
require 'hooks'
|
2
|
+
|
3
|
+
module Tochtli
|
4
|
+
class BaseController
|
5
|
+
extend Uber::InheritableAttribute
|
6
|
+
include Hooks
|
7
|
+
include Hooks::InstanceHooks
|
8
|
+
|
9
|
+
inheritable_attr :routing_keys
|
10
|
+
inheritable_attr :message_handlers
|
11
|
+
inheritable_attr :work_pool_size
|
12
|
+
|
13
|
+
self.work_pool_size = 1 # default work pool size per controller instance
|
14
|
+
|
15
|
+
attr_reader :logger, :env, :message, :delivery_info
|
16
|
+
|
17
|
+
# Each controller can overwrite the queue name (default: controller.name.underscore)
|
18
|
+
inheritable_attr :queue_name
|
19
|
+
|
20
|
+
# Custom options for controller queue and exchange
|
21
|
+
inheritable_attr :queue_durable
|
22
|
+
self.queue_durable = true
|
23
|
+
|
24
|
+
inheritable_attr :queue_auto_delete
|
25
|
+
self.queue_auto_delete = false
|
26
|
+
|
27
|
+
inheritable_attr :queue_exclusive
|
28
|
+
self.queue_exclusive = false
|
29
|
+
|
30
|
+
inheritable_attr :exchange_type
|
31
|
+
self.exchange_type = :topic
|
32
|
+
|
33
|
+
inheritable_attr :exchange_name # read from configuration by default
|
34
|
+
|
35
|
+
inheritable_attr :exchange_durable
|
36
|
+
self.exchange_durable = true
|
37
|
+
|
38
|
+
# Message dispatcher created on start
|
39
|
+
inheritable_attr :dispatcher
|
40
|
+
|
41
|
+
define_hooks :before_setup, :after_setup,
|
42
|
+
:before_start, :after_start,
|
43
|
+
:before_stop, :after_stop,
|
44
|
+
:before_restart, :after_restart
|
45
|
+
|
46
|
+
class << self
|
47
|
+
def inherited(controller)
|
48
|
+
controller.routing_keys = Set.new
|
49
|
+
controller.message_handlers = Array.new
|
50
|
+
controller.queue_name = controller.name.underscore.gsub('::', '/')
|
51
|
+
ControllerManager.register(controller)
|
52
|
+
end
|
53
|
+
|
54
|
+
def bind(*routing_keys)
|
55
|
+
self.routing_keys.merge(routing_keys)
|
56
|
+
end
|
57
|
+
|
58
|
+
def on(message_class, method_name=nil, opts={}, &block)
|
59
|
+
if method_name.is_a?(Hash)
|
60
|
+
opts = method_name
|
61
|
+
method_name = nil
|
62
|
+
end
|
63
|
+
method = method_name ? method_name : block
|
64
|
+
raise ArgumentError, "Method name or block must be given" unless method
|
65
|
+
|
66
|
+
raise ArgumentError, "Message class expected, got: #{message_class}" unless message_class < Tochtli::Message
|
67
|
+
|
68
|
+
routing_key = opts[:routing_key] || message_class.routing_key
|
69
|
+
raise "Topic not set for message: #{message_class}" unless routing_key
|
70
|
+
|
71
|
+
self.message_handlers << MessageRoute.new(message_class, method, routing_key)
|
72
|
+
end
|
73
|
+
|
74
|
+
def off(routing_key)
|
75
|
+
self.message_handlers.delete_if {|route| route.routing_key == routing_key }
|
76
|
+
end
|
77
|
+
|
78
|
+
|
79
|
+
def setup(rabbit_connection, cache=nil, logger=nil)
|
80
|
+
run_hook :before_setup, rabbit_connection
|
81
|
+
self.dispatcher = Dispatcher.new(self, rabbit_connection, cache, logger || Tochtli.logger)
|
82
|
+
run_hook :after_setup, rabbit_connection
|
83
|
+
end
|
84
|
+
|
85
|
+
def start(queue_name=nil, routing_keys=nil, initial_env={})
|
86
|
+
run_hook :before_start, queue_name, initial_env
|
87
|
+
self.dispatcher.start(queue_name || self.queue_name, routing_keys || self.routing_keys, initial_env)
|
88
|
+
run_hook :after_start, queue_name, initial_env
|
89
|
+
end
|
90
|
+
|
91
|
+
def set_up?
|
92
|
+
!!self.dispatcher
|
93
|
+
end
|
94
|
+
|
95
|
+
def started?(queue_name=nil)
|
96
|
+
self.dispatcher && self.dispatcher.started?(queue_name)
|
97
|
+
end
|
98
|
+
|
99
|
+
def stop(options={})
|
100
|
+
if started?
|
101
|
+
queues = self.dispatcher.queues
|
102
|
+
run_hook :before_stop, queues
|
103
|
+
|
104
|
+
self.dispatcher.shutdown(options)
|
105
|
+
self.dispatcher = nil
|
106
|
+
|
107
|
+
run_hook :after_stop, queues
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def restart(options={})
|
112
|
+
if started?
|
113
|
+
queues = self.dispatcher.queues
|
114
|
+
run_hook :before_restart, queues
|
115
|
+
self.dispatcher.restart options
|
116
|
+
run_hook :after_restart, queues
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def find_message_route(routing_key)
|
121
|
+
raise "Routing not set up" if self.message_handlers.empty?
|
122
|
+
self.message_handlers.find {|handler| handler.pattern =~ routing_key }
|
123
|
+
end
|
124
|
+
|
125
|
+
def create_queue(rabbit_connection, queue_name=nil, routing_keys=nil)
|
126
|
+
queue_name = self.queue_name unless queue_name
|
127
|
+
routing_keys = self.routing_keys unless routing_keys
|
128
|
+
channel = rabbit_connection.create_channel(self.work_pool_size)
|
129
|
+
exchange_name = self.exchange_name || rabbit_connection.exchange_name
|
130
|
+
exchange = channel.send(self.exchange_type, exchange_name, durable: self.exchange_durable)
|
131
|
+
queue = channel.queue(queue_name,
|
132
|
+
durable: self.queue_durable,
|
133
|
+
exclusive: self.queue_exclusive,
|
134
|
+
auto_delete: self.queue_auto_delete)
|
135
|
+
|
136
|
+
routing_keys.each do |routing_key|
|
137
|
+
queue.bind(exchange, routing_key: routing_key)
|
138
|
+
end
|
139
|
+
|
140
|
+
queue
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
public
|
145
|
+
|
146
|
+
def initialize(rabbit_connection, cache, logger)
|
147
|
+
@rabbit_connection = rabbit_connection
|
148
|
+
@cache = cache
|
149
|
+
@logger = logger
|
150
|
+
end
|
151
|
+
|
152
|
+
def process_message(env)
|
153
|
+
@env = env
|
154
|
+
@action = env[:action]
|
155
|
+
@message = env[:message]
|
156
|
+
@delivery_info = env[:delivery_info]
|
157
|
+
|
158
|
+
if @action.is_a?(Proc)
|
159
|
+
instance_eval(&@action)
|
160
|
+
else
|
161
|
+
send @action
|
162
|
+
end
|
163
|
+
|
164
|
+
ensure
|
165
|
+
@env, @message, @delivery_info = nil
|
166
|
+
end
|
167
|
+
|
168
|
+
def reply(reply_message, reply_to=nil, message_id=nil)
|
169
|
+
if @message
|
170
|
+
reply_to ||= @message.properties.reply_to
|
171
|
+
message_id ||= @message.id
|
172
|
+
end
|
173
|
+
|
174
|
+
raise "The 'reply_to' queue name is not specified" unless reply_to
|
175
|
+
|
176
|
+
logger.debug "\tSending reply on #{message_id} to #{reply_to}: #{reply_message.inspect}."
|
177
|
+
|
178
|
+
@rabbit_connection.publish(reply_to,
|
179
|
+
reply_message,
|
180
|
+
correlation_id: message_id)
|
181
|
+
end
|
182
|
+
|
183
|
+
def rabbit_connection
|
184
|
+
self.class.dispatcher.rabbit_connection if self.class.set_up?
|
185
|
+
end
|
186
|
+
|
187
|
+
class MessageRoute < Struct.new(:message_class, :action, :routing_key, :pattern)
|
188
|
+
def initialize(message_class, action, routing_key)
|
189
|
+
super message_class, action, routing_key, KeyPattern.new(routing_key)
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
class Dispatcher
|
194
|
+
attr_reader :controller_class, :rabbit_connection, :cache, :logger
|
195
|
+
|
196
|
+
def initialize(controller_class, rabbit_connection, cache, logger)
|
197
|
+
@controller_class = controller_class
|
198
|
+
@rabbit_connection = rabbit_connection
|
199
|
+
@cache = cache
|
200
|
+
@logger = logger
|
201
|
+
@application = Tochtli.application.to_app
|
202
|
+
@queues = {}
|
203
|
+
@process_counter = ProcessCounter.new
|
204
|
+
@initial_env = nil
|
205
|
+
end
|
206
|
+
|
207
|
+
def start(queue_name, routing_keys, initial_env={})
|
208
|
+
subscribe_queue(queue_name, routing_keys, initial_env)
|
209
|
+
end
|
210
|
+
|
211
|
+
def restart(options={})
|
212
|
+
queues = @queues.dup
|
213
|
+
|
214
|
+
shutdown options
|
215
|
+
|
216
|
+
queues.each do |queue_name, queue_opts|
|
217
|
+
start queue_name, queue_opts[:initial_env]
|
218
|
+
end
|
219
|
+
end
|
220
|
+
|
221
|
+
def process_message(delivery_info, properties, payload, initial_env)
|
222
|
+
register_process_start
|
223
|
+
|
224
|
+
env = initial_env.merge(
|
225
|
+
delivery_info: delivery_info,
|
226
|
+
properties: properties,
|
227
|
+
payload: payload,
|
228
|
+
controller_class: controller_class,
|
229
|
+
rabbit_connection: rabbit_connection,
|
230
|
+
cache: cache,
|
231
|
+
logger: logger
|
232
|
+
)
|
233
|
+
|
234
|
+
@application.call(env)
|
235
|
+
|
236
|
+
rescue Exception => ex
|
237
|
+
logger.error "\nUNEXPECTED EXCEPTION: #{ex.class.name} (#{ex.message})"
|
238
|
+
logger.error ex.backtrace.join("\n")
|
239
|
+
false
|
240
|
+
ensure
|
241
|
+
register_process_end
|
242
|
+
end
|
243
|
+
|
244
|
+
def subscribe_queue(queue_name, routing_keys, initial_env={})
|
245
|
+
queue = controller_class.create_queue(@rabbit_connection, queue_name, routing_keys)
|
246
|
+
consumer = queue.subscribe do |delivery_info, metadata, payload|
|
247
|
+
process_message delivery_info, metadata, payload, initial_env
|
248
|
+
end
|
249
|
+
|
250
|
+
@queues[queue_name] = {
|
251
|
+
queue: queue,
|
252
|
+
consumer: consumer,
|
253
|
+
initial_env: initial_env
|
254
|
+
}
|
255
|
+
end
|
256
|
+
|
257
|
+
# Performs a graceful shutdown of dispatcher i.e. waits for all processes to end.
|
258
|
+
# If timeout is reached, forces the shutdown. Useful with dynamic reconfiguration of work pool size.
|
259
|
+
def shutdown(options={})
|
260
|
+
wait_for_processes options.fetch(:timeout, 15)
|
261
|
+
stop
|
262
|
+
end
|
263
|
+
|
264
|
+
def stop(queues=nil)
|
265
|
+
@queues.each_value { |queue_opts| queue_opts[:consumer].cancel }
|
266
|
+
rescue Bunny::ConnectionClosedError
|
267
|
+
# ignore closed connection error
|
268
|
+
ensure
|
269
|
+
@queues = {}
|
270
|
+
end
|
271
|
+
|
272
|
+
def started?(queue_name=nil)
|
273
|
+
if queue_name
|
274
|
+
@queues.has_key?(queue_name)
|
275
|
+
else
|
276
|
+
!@queues.empty?
|
277
|
+
end
|
278
|
+
end
|
279
|
+
|
280
|
+
def queues
|
281
|
+
@queues.map { |_, qh| qh[:queue] }
|
282
|
+
end
|
283
|
+
|
284
|
+
protected
|
285
|
+
|
286
|
+
def register_process_start
|
287
|
+
@process_counter.increment
|
288
|
+
end
|
289
|
+
|
290
|
+
def register_process_end
|
291
|
+
@process_counter.decrement
|
292
|
+
end
|
293
|
+
|
294
|
+
def wait_for_processes(timeout)
|
295
|
+
@process_counter.wait(timeout)
|
296
|
+
end
|
297
|
+
|
298
|
+
class ProcessCounter
|
299
|
+
include MonitorMixin
|
300
|
+
|
301
|
+
def initialize
|
302
|
+
super
|
303
|
+
@count = 0
|
304
|
+
@cv = new_cond
|
305
|
+
end
|
306
|
+
|
307
|
+
def increment
|
308
|
+
synchronize do
|
309
|
+
@count += 1
|
310
|
+
end
|
311
|
+
end
|
312
|
+
|
313
|
+
def decrement
|
314
|
+
synchronize do
|
315
|
+
@count -= 1 if @count > 0
|
316
|
+
@cv.signal if @count == 0
|
317
|
+
end
|
318
|
+
end
|
319
|
+
|
320
|
+
def wait(timeout)
|
321
|
+
synchronize do
|
322
|
+
@cv.wait(timeout) unless @count == 0
|
323
|
+
end
|
324
|
+
end
|
325
|
+
end
|
326
|
+
end
|
327
|
+
|
328
|
+
class KeyPattern
|
329
|
+
ASTERIX_EXP = '[a-zA-Z0-9]+'
|
330
|
+
HASH_EXP = '[a-zA-Z0-9\.]*'
|
331
|
+
|
332
|
+
def initialize(pattern)
|
333
|
+
@str = pattern
|
334
|
+
@simple = !pattern.include?('*') && !pattern.include?('#')
|
335
|
+
if @simple
|
336
|
+
@pattern = pattern
|
337
|
+
else
|
338
|
+
@pattern = Regexp.new('^' + pattern.gsub('.', '\\.').
|
339
|
+
gsub('*', ASTERIX_EXP).gsub(/(\\\.)?#(\\\.)?/, HASH_EXP) + '$')
|
340
|
+
end
|
341
|
+
end
|
342
|
+
|
343
|
+
def =~(key)
|
344
|
+
if @simple
|
345
|
+
@pattern == key
|
346
|
+
else
|
347
|
+
@pattern =~ key
|
348
|
+
end
|
349
|
+
end
|
350
|
+
|
351
|
+
def !~(key)
|
352
|
+
!(self =~ key)
|
353
|
+
end
|
354
|
+
|
355
|
+
def to_s
|
356
|
+
@str
|
357
|
+
end
|
358
|
+
end
|
359
|
+
end
|
360
|
+
end
|