tochtli 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +7 -0
  2. data/.travis.yml +14 -0
  3. data/Gemfile +32 -0
  4. data/History.md +138 -0
  5. data/README.md +46 -0
  6. data/Rakefile +50 -0
  7. data/VERSION +1 -0
  8. data/assets/communication.png +0 -0
  9. data/assets/layers.png +0 -0
  10. data/examples/01-screencap-service/Gemfile +3 -0
  11. data/examples/01-screencap-service/README.md +5 -0
  12. data/examples/01-screencap-service/client.rb +15 -0
  13. data/examples/01-screencap-service/common.rb +15 -0
  14. data/examples/01-screencap-service/server.rb +26 -0
  15. data/examples/02-log-analyzer/Gemfile +3 -0
  16. data/examples/02-log-analyzer/README.md +5 -0
  17. data/examples/02-log-analyzer/client.rb +95 -0
  18. data/examples/02-log-analyzer/common.rb +33 -0
  19. data/examples/02-log-analyzer/sample.log +10001 -0
  20. data/examples/02-log-analyzer/server.rb +133 -0
  21. data/lib/tochtli.rb +177 -0
  22. data/lib/tochtli/active_record_connection_cleaner.rb +9 -0
  23. data/lib/tochtli/application.rb +135 -0
  24. data/lib/tochtli/base_client.rb +135 -0
  25. data/lib/tochtli/base_controller.rb +360 -0
  26. data/lib/tochtli/controller_manager.rb +99 -0
  27. data/lib/tochtli/engine.rb +15 -0
  28. data/lib/tochtli/message.rb +114 -0
  29. data/lib/tochtli/rabbit_client.rb +36 -0
  30. data/lib/tochtli/rabbit_connection.rb +249 -0
  31. data/lib/tochtli/reply_queue.rb +129 -0
  32. data/lib/tochtli/simple_validation.rb +23 -0
  33. data/lib/tochtli/test.rb +9 -0
  34. data/lib/tochtli/test/client.rb +28 -0
  35. data/lib/tochtli/test/controller.rb +66 -0
  36. data/lib/tochtli/test/integration.rb +78 -0
  37. data/lib/tochtli/test/memory_cache.rb +22 -0
  38. data/lib/tochtli/test/test_case.rb +191 -0
  39. data/lib/tochtli/test/test_unit.rb +22 -0
  40. data/lib/tochtli/version.rb +3 -0
  41. data/log_generator.rb +11 -0
  42. data/test/base_client_test.rb +68 -0
  43. data/test/controller_functional_test.rb +87 -0
  44. data/test/controller_integration_test.rb +274 -0
  45. data/test/controller_manager_test.rb +75 -0
  46. data/test/dummy/Rakefile +7 -0
  47. data/test/dummy/config/application.rb +36 -0
  48. data/test/dummy/config/boot.rb +4 -0
  49. data/test/dummy/config/database.yml +3 -0
  50. data/test/dummy/config/environment.rb +5 -0
  51. data/test/dummy/config/rabbit.yml +4 -0
  52. data/test/dummy/db/.gitkeep +0 -0
  53. data/test/dummy/log/.gitkeep +0 -0
  54. data/test/key_matcher_test.rb +100 -0
  55. data/test/log/.gitkeep +0 -0
  56. data/test/message_test.rb +80 -0
  57. data/test/rabbit_client_test.rb +71 -0
  58. data/test/rabbit_connection_test.rb +151 -0
  59. data/test/test_helper.rb +32 -0
  60. data/test/version_test.rb +8 -0
  61. data/tochtli.gemspec +129 -0
  62. metadata +259 -0
@@ -0,0 +1,99 @@
1
+ require 'singleton'
2
+
3
+ module Tochtli
4
+ class ControllerManager
5
+ include Singleton
6
+
7
+ attr_reader :rabbit_connection, :cache, :logger
8
+
9
+ def initialize
10
+ @controller_classes = Set.new
11
+ end
12
+
13
+ def register(controller_class)
14
+ raise ArgumentError, "Controller expected, got: #{controller_class}" unless controller_class.is_a?(Class) && controller_class < Tochtli::BaseController
15
+ @controller_classes << controller_class
16
+ end
17
+
18
+ def setup(options={})
19
+ @logger = options.fetch(:logger, Tochtli.logger)
20
+ @cache = options.fetch(:cache, Tochtli.cache)
21
+ @rabbit_connection = options[:connection]
22
+
23
+ unless @rabbit_connection
24
+ @rabbit_connection = RabbitConnection.open(options[:config])
25
+ end
26
+ end
27
+
28
+ def start(*controllers)
29
+ options = controllers.extract_options!
30
+ setup_options = options.except!(:logger, :cache, :connection)
31
+ queue_name = options.delete(:queue_name)
32
+ routing_keys = options.delete(:routing_keys)
33
+ initial_env = options.delete(:env) || {}
34
+
35
+ setup(setup_options) unless set_up?
36
+
37
+ if controllers.empty? || controllers.include?(:all)
38
+ controllers = @controller_classes
39
+ end
40
+
41
+ controllers.each do |controller_class|
42
+ raise ArgumentError, "Controller expected, got: #{controller_class.inspect}" unless controller_class.is_a?(Class) && controller_class < Tochtli::BaseController
43
+ unless controller_class.started?(queue_name)
44
+ @logger.info "Starting #{controller_class}..." if @logger
45
+ controller_class.setup(@rabbit_connection, @cache, @logger) unless controller_class.set_up?
46
+ controller_class.start queue_name, routing_keys, initial_env
47
+ end
48
+ end
49
+ end
50
+
51
+ def stop
52
+ @controller_classes.each do |controller_class|
53
+ if controller_class.started?
54
+ @logger.info "Stopping #{controller_class}..." if @logger
55
+ controller_class.stop
56
+ end
57
+ end
58
+ @rabbit_connection = nil
59
+ end
60
+
61
+ def restart(options={})
62
+ options[:rabbit_connection] ||= @rabbit_connection
63
+ options[:logger] ||= @logger
64
+ options[:cache] ||= @cache
65
+
66
+ setup options
67
+ restart_active_controllers
68
+ end
69
+
70
+ def set_up?
71
+ !@rabbit_connection.nil?
72
+ end
73
+
74
+ def running?
75
+ @rabbit_connection && @rabbit_connection.open?
76
+ end
77
+
78
+ protected
79
+
80
+ def restart_active_controllers
81
+ @controller_classes.each do |controller_class|
82
+ if controller_class.started?
83
+ @logger.info "Restarting #{controller_class}..." if @logger
84
+ controller_class.restart
85
+ end
86
+ end
87
+ end
88
+
89
+ class << self
90
+ def method_missing(method, *args)
91
+ if instance.respond_to?(method)
92
+ instance.send(method, *args)
93
+ else
94
+ super
95
+ end
96
+ end
97
+ end
98
+ end
99
+ end
@@ -0,0 +1,15 @@
1
+ module Tochtli
2
+ class Engine < ::Rails::Engine
3
+
4
+ initializer :eager_load_messages, :before => :bootstrap_hook do
5
+ Tochtli.eager_load_service_messages
6
+ end
7
+
8
+ initializer :use_active_record_connection_release do
9
+ ActiveSupport.on_load(:active_record) do
10
+ Tochtli.application.middlewares.use Tochtli::ActiveRecordConnectionCleaner
11
+ end
12
+ end
13
+
14
+ end
15
+ end
@@ -0,0 +1,114 @@
1
+ require 'securerandom'
2
+
3
+ module Tochtli
4
+ class Message
5
+ extend Uber::InheritableAttr
6
+ include Virtus.model
7
+ include Tochtli::SimpleValidation
8
+
9
+ inheritable_attr :routing_key
10
+ inheritable_attr :extra_attributes_policy
11
+
12
+ attr_reader :id, :properties
13
+
14
+ def self.route_to(routing_key=nil, &block)
15
+ self.routing_key = routing_key || block
16
+ end
17
+
18
+ # Compatibility with version 0.3
19
+ def self.attributes(*attributes)
20
+ options = attributes.extract_options!
21
+ required = options.fetch(:validate, true)
22
+
23
+ attributes.each do |name|
24
+ attribute name, String, required: required
25
+ end
26
+ end
27
+
28
+ def self.required_attributes(*attributes)
29
+ options = attributes.extract_options!
30
+ self.attributes *attributes, options.merge(validate: true)
31
+ end
32
+
33
+ def self.optional_attributes(*attributes)
34
+ options = attributes.extract_options!
35
+ self.attributes *attributes, options.merge(validate: false)
36
+ end
37
+
38
+ def self.ignore_extra_attributes
39
+ self.extra_attributes_policy = :ignore
40
+ end
41
+
42
+ def initialize(attributes={}, properties=nil)
43
+ super attributes || {}
44
+
45
+ @properties = properties
46
+ @id = properties.message_id if properties
47
+ @id ||= self.class.generate_id
48
+
49
+ store_extra_attributes(attributes)
50
+ end
51
+
52
+ def attributes=(attributes)
53
+ super
54
+ store_extra_attributes(attributes)
55
+ end
56
+
57
+ def store_extra_attributes(attributes)
58
+ @extra_attributes ||= {}
59
+ if attributes
60
+ attributes.each do |name, value|
61
+ unless allowed_writer_methods.include?("#{name}=")
62
+ @extra_attributes[name] = value
63
+ end
64
+ end
65
+ end
66
+ end
67
+
68
+ def validate_extra_attributes
69
+ if self.class.extra_attributes_policy != :ignore && !@extra_attributes.empty?
70
+ add_error "Unexpected attributes: #{@extra_attributes.keys.map(&:to_s).join(', ')}"
71
+ end
72
+ end
73
+
74
+ def validate_attributes_presence
75
+ nil_attributes = attribute_set.select { |a| a.required? && self[a.name].nil? }.map(&:name)
76
+ unless nil_attributes.empty?
77
+ add_error "Required attributes: #{nil_attributes.map(&:to_s).join(', ')} not specified"
78
+ end
79
+ end
80
+
81
+ def validate
82
+ validate_extra_attributes
83
+ validate_attributes_presence
84
+ end
85
+
86
+ def self.generate_id
87
+ SecureRandom.uuid
88
+ end
89
+
90
+ def routing_key
91
+ if self.class.routing_key.is_a?(Proc)
92
+ self.instance_eval(&self.class.routing_key)
93
+ else
94
+ self.class.routing_key
95
+ end
96
+ end
97
+
98
+ def to_hash
99
+ attributes.inject({}) do |hash, (name, value)|
100
+ value = value.map(&:to_hash) if value.is_a?(Array)
101
+ hash[name] = value
102
+ hash
103
+ end
104
+ end
105
+
106
+ def to_json
107
+ JSON.dump(to_hash)
108
+ end
109
+ end
110
+
111
+ class ErrorMessage < Message
112
+ attributes :error, :message
113
+ end
114
+ end
@@ -0,0 +1,36 @@
1
+ module Tochtli
2
+ class RabbitClient
3
+
4
+ attr_reader :rabbit_connection
5
+
6
+ def initialize(rabbit_connection=nil, logger=nil)
7
+ if rabbit_connection
8
+ @rabbit_connection = rabbit_connection
9
+ else
10
+ @rabbit_connection = Tochtli::RabbitConnection.open(nil, logger: logger)
11
+ end
12
+ @logger = logger || @rabbit_connection.logger
13
+ end
14
+
15
+ def publish(message, options={})
16
+ raise InvalidMessageError.new(message.errors.join(", "), message) if message.invalid?
17
+
18
+ @logger.debug "[#{Time.now} AMQP] Publishing message #{message.id} to #{message.routing_key}"
19
+
20
+ reply_queue = @rabbit_connection.reply_queue
21
+ options[:reply_to] = reply_queue.name
22
+ if (message_handler = options[:handler])
23
+ reply_queue.register_message_handler message, message_handler, options[:timeout]
24
+ end
25
+ @rabbit_connection.publish message.routing_key, message, options
26
+ end
27
+
28
+ def wait_for_confirms
29
+ @rabbit_connection.channel.wait_for_confirms
30
+ end
31
+
32
+ def reply_queue(*args)
33
+ rabbit_connection.reply_queue(*args)
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,249 @@
1
+ require 'bunny'
2
+ require 'securerandom'
3
+
4
+ module Tochtli
5
+ class RabbitConnection
6
+ attr_accessor :connection
7
+ attr_reader :logger, :exchange_name
8
+
9
+ cattr_accessor :connections
10
+ self.connections = {}
11
+
12
+ private_class_method :new
13
+
14
+ DEFAULT_CONNECTION_NAME = 'default'
15
+
16
+ def initialize(config = nil, channel_pool=nil)
17
+ @config = config.is_a?(RabbitConnection::Config) ? config : RabbitConnection::Config.load(nil, config)
18
+ @exchange_name = @config.delete(:exchange_name)
19
+ @work_pool_size = @config.delete(:work_pool_size)
20
+ @logger = @config.delete(:logger) || Tochtli.logger
21
+ @channel_pool = channel_pool ? channel_pool : Hash.new
22
+ end
23
+
24
+ def self.open(name=nil, config=nil)
25
+ name ||= defined?(Rails) ? Rails.env : DEFAULT_CONNECTION_NAME
26
+ raise ArgumentError, "RabbitMQ configuration name not specified" if !name && !ENV.has_key?('RABBITMQ_URL')
27
+ connection = self.connections[name.to_sym]
28
+ if !connection || !connection.open?
29
+ config = config.is_a?(RabbitConnection::Config) ? config : RabbitConnection::Config.load(name, config)
30
+ connection = new(config)
31
+ connection.connect
32
+ self.connections[name.to_sym] = connection
33
+ end
34
+
35
+ if block_given?
36
+ yield connection
37
+ close name
38
+ else
39
+ connection
40
+ end
41
+ end
42
+
43
+ def self.close(name=nil)
44
+ name ||= defined?(Rails) ? Rails.env : nil
45
+ raise ArgumentError, "RabbitMQ configuration name not specified" unless name
46
+ connection = self.connections.delete(name.to_sym)
47
+ connection.disconnect if connection && connection.open?
48
+ end
49
+
50
+ def connect(opts={})
51
+ return if open?
52
+
53
+ defaults = {}
54
+ unless opts[:logger]
55
+ defaults[:logger] = @logger.dup
56
+ defaults[:logger].level = Tochtli.debug_bunny ? Logger::DEBUG : Logger::WARN
57
+ end
58
+
59
+ setup_bunny_connection(defaults.merge(opts))
60
+
61
+ if block_given?
62
+ yield
63
+ disconnect if open?
64
+ end
65
+ end
66
+
67
+ def disconnect
68
+ @connection.close if @connection
69
+ rescue Bunny::ClientTimeout
70
+ false
71
+ ensure
72
+ @channel_pool.clear
73
+ @connection = nil
74
+ @reply_queue = nil
75
+ end
76
+
77
+ def open?
78
+ @connection && @connection.open?
79
+ end
80
+
81
+ def setup_bunny_connection(opts={})
82
+ @connection = Bunny.new(@config, opts)
83
+ @connection.start
84
+ rescue Bunny::TCPConnectionFailed => ex
85
+ connection_url = "amqp://#{@connection.user}@#{@connection.host}:#{@connection.port}/#{@connection.vhost}"
86
+ raise ConnectionFailed.new("Unable to connect to: '#{connection_url}' (#{ex.message})")
87
+ end
88
+
89
+ def create_reply_queue
90
+ Tochtli::ReplyQueue.new(self, @logger)
91
+ end
92
+
93
+ def reply_queue
94
+ @reply_queue ||= create_reply_queue
95
+ end
96
+
97
+ def exchange(thread=Thread.current)
98
+ channel_wrap(thread).exchange
99
+ end
100
+
101
+ def channel(thread=Thread.current)
102
+ channel_wrap(thread).channel
103
+ end
104
+
105
+ def queue(name, routing_keys=[], options={})
106
+ queue = channel.queue(name, {durable: true}.merge(options))
107
+ routing_keys.each do |routing_key|
108
+ queue.bind(exchange, routing_key: routing_key)
109
+ end
110
+ queue
111
+ end
112
+
113
+ def queue_exists?(name)
114
+ @connection.queue_exists?(name)
115
+ end
116
+
117
+ def ack(delivery_tag)
118
+ channel.ack(delivery_tag, false)
119
+ end
120
+
121
+ def publish(routing_key, message, options={})
122
+ begin
123
+ payload = message.to_json
124
+ rescue Exception
125
+ logger.error "Unable to serialize message: #{message.inspect}"
126
+ logger.error $!
127
+ raise "Unable to serialize message to JSON: #{$!}"
128
+ end
129
+
130
+ exchange.publish(payload, {
131
+ routing_key: routing_key,
132
+ persistent: true,
133
+ mandatory: true,
134
+ timestamp: Time.now.to_i,
135
+ message_id: message.id,
136
+ type: message.class.name.underscore,
137
+ content_type: "application/json"
138
+ }.merge(options))
139
+ end
140
+
141
+ def create_channel(consumer_pool_size = 1)
142
+ @connection.create_channel(nil, consumer_pool_size).tap do |channel|
143
+ channel.confirm_select # use publisher confirmations
144
+ end
145
+ end
146
+
147
+ def create_exchange(channel)
148
+ channel.topic(@exchange_name, durable: true)
149
+ end
150
+
151
+ private
152
+
153
+ def on_return(return_info, properties, payload)
154
+ unless properties[:correlation_id]
155
+ error_message = "Message #{properties[:message_id]} dropped: #{return_info[:reply_text]} [#{return_info[:reply_code]}]"
156
+ reply_queue.handle_reply MessageDropped.new(error_message, payload), properties[:message_id]
157
+ else # a reply dropped - client reply queue probably does not exist any more
158
+ logger.debug "Reply on message #{properties[:correlation_id]} dropped: #{return_info[:reply_text]} [#{return_info[:reply_code]}]"
159
+ end
160
+ rescue
161
+ logger.error "Internal error (on_return): #{$!}"
162
+ logger.error $!.backtrace.join("\n")
163
+ end
164
+
165
+ def create_channel_wrap(thread=Thread.current)
166
+ raise ConnectionFailed.new("Channel already created for thread #{thread.object_id}") if @channel_pool[thread.object_id]
167
+ raise ConnectionFailed.new("Unable to create channel. Connection lost.") unless @connection
168
+
169
+ channel = create_channel(@work_pool_size)
170
+ exchange = create_exchange(channel)
171
+ exchange.on_return &method(:on_return)
172
+
173
+ channel_wrap = ChannelWrap.new(channel, exchange)
174
+ @channel_pool[thread.object_id] = channel_wrap
175
+
176
+ channel_wrap
177
+ rescue Bunny::PreconditionFailed => ex
178
+ raise ConnectionFailed.new("Unable create exchange: '#{@exchange_name}': #{ex.message}")
179
+ end
180
+
181
+ def channel_wrap(thread=Thread.current)
182
+ channel_wrap = @channel_pool[thread.object_id]
183
+ if channel_wrap && channel_wrap.channel.active
184
+ channel_wrap
185
+ else
186
+ @channel_pool.delete(thread.object_id) # ensure inactive channel s not cached
187
+ create_channel_wrap(thread)
188
+ end
189
+ end
190
+
191
+ def generate_id
192
+ SecureRandom.uuid
193
+ end
194
+
195
+ class ChannelWrap
196
+ attr_reader :channel, :exchange
197
+
198
+ def initialize(channel, exchange)
199
+ @channel = channel
200
+ @exchange = exchange
201
+ end
202
+ end
203
+
204
+ class Config < Hash
205
+ DEFAULTS = {
206
+ :exchange_name => "puzzleflow.services",
207
+ :work_pool_size => 1,
208
+ :automatically_recover => true,
209
+ :network_recovery_interval => 1
210
+ }
211
+
212
+ def self.load(name, config=nil)
213
+ config = case config
214
+ when String
215
+ YAML.load_file(config).symbolize_keys
216
+ when Hash
217
+ config.symbolize_keys
218
+ when nil
219
+ {}
220
+ else
221
+ raise "Unexpected configuration: #{config.inspect}, Hash or String expected."
222
+ end
223
+
224
+ defaults = DEFAULTS
225
+
226
+ if defined?(Rails) && Rails.root
227
+ config_path = Rails.root.join('config/rabbit.yml')
228
+ if config_path.exist?
229
+ rails_config = YAML.load_file(config_path)
230
+ raise "Unexpected rabbit.yml: #{rails_config.inspect}, Hash expected." unless rails_config.is_a?(Hash)
231
+ rails_config = rails_config.symbolize_keys
232
+ unless rails_config[:host] # backward compatibility
233
+ rails_config = rails_config[name.to_sym]
234
+ raise "RabbitMQ '#{name}' configuration not set in rabbit.yml" unless rails_config
235
+ else
236
+ warn "DEPRECATION WARNING: rabbit.yml should define different configurations for Rails environments (like database.yml). Please update your configuration file: #{config_path}."
237
+ end
238
+ defaults = defaults.merge(rails_config.symbolize_keys)
239
+ end
240
+ end
241
+
242
+ new.merge!(defaults.merge(config))
243
+ end
244
+ end
245
+
246
+ class ConnectionFailed < StandardError
247
+ end
248
+ end
249
+ end