tochtli 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +7 -0
  2. data/.travis.yml +14 -0
  3. data/Gemfile +32 -0
  4. data/History.md +138 -0
  5. data/README.md +46 -0
  6. data/Rakefile +50 -0
  7. data/VERSION +1 -0
  8. data/assets/communication.png +0 -0
  9. data/assets/layers.png +0 -0
  10. data/examples/01-screencap-service/Gemfile +3 -0
  11. data/examples/01-screencap-service/README.md +5 -0
  12. data/examples/01-screencap-service/client.rb +15 -0
  13. data/examples/01-screencap-service/common.rb +15 -0
  14. data/examples/01-screencap-service/server.rb +26 -0
  15. data/examples/02-log-analyzer/Gemfile +3 -0
  16. data/examples/02-log-analyzer/README.md +5 -0
  17. data/examples/02-log-analyzer/client.rb +95 -0
  18. data/examples/02-log-analyzer/common.rb +33 -0
  19. data/examples/02-log-analyzer/sample.log +10001 -0
  20. data/examples/02-log-analyzer/server.rb +133 -0
  21. data/lib/tochtli.rb +177 -0
  22. data/lib/tochtli/active_record_connection_cleaner.rb +9 -0
  23. data/lib/tochtli/application.rb +135 -0
  24. data/lib/tochtli/base_client.rb +135 -0
  25. data/lib/tochtli/base_controller.rb +360 -0
  26. data/lib/tochtli/controller_manager.rb +99 -0
  27. data/lib/tochtli/engine.rb +15 -0
  28. data/lib/tochtli/message.rb +114 -0
  29. data/lib/tochtli/rabbit_client.rb +36 -0
  30. data/lib/tochtli/rabbit_connection.rb +249 -0
  31. data/lib/tochtli/reply_queue.rb +129 -0
  32. data/lib/tochtli/simple_validation.rb +23 -0
  33. data/lib/tochtli/test.rb +9 -0
  34. data/lib/tochtli/test/client.rb +28 -0
  35. data/lib/tochtli/test/controller.rb +66 -0
  36. data/lib/tochtli/test/integration.rb +78 -0
  37. data/lib/tochtli/test/memory_cache.rb +22 -0
  38. data/lib/tochtli/test/test_case.rb +191 -0
  39. data/lib/tochtli/test/test_unit.rb +22 -0
  40. data/lib/tochtli/version.rb +3 -0
  41. data/log_generator.rb +11 -0
  42. data/test/base_client_test.rb +68 -0
  43. data/test/controller_functional_test.rb +87 -0
  44. data/test/controller_integration_test.rb +274 -0
  45. data/test/controller_manager_test.rb +75 -0
  46. data/test/dummy/Rakefile +7 -0
  47. data/test/dummy/config/application.rb +36 -0
  48. data/test/dummy/config/boot.rb +4 -0
  49. data/test/dummy/config/database.yml +3 -0
  50. data/test/dummy/config/environment.rb +5 -0
  51. data/test/dummy/config/rabbit.yml +4 -0
  52. data/test/dummy/db/.gitkeep +0 -0
  53. data/test/dummy/log/.gitkeep +0 -0
  54. data/test/key_matcher_test.rb +100 -0
  55. data/test/log/.gitkeep +0 -0
  56. data/test/message_test.rb +80 -0
  57. data/test/rabbit_client_test.rb +71 -0
  58. data/test/rabbit_connection_test.rb +151 -0
  59. data/test/test_helper.rb +32 -0
  60. data/test/version_test.rb +8 -0
  61. data/tochtli.gemspec +129 -0
  62. metadata +259 -0
@@ -0,0 +1,133 @@
1
+ require_relative 'common'
2
+
3
+ Thread.abort_on_exception = true
4
+
5
+ Tochtli.logger.progname = 'SERVER'
6
+
7
+ module LogAnalyzer
8
+
9
+ class LogController < Tochtli::BaseController
10
+ bind 'log.analyzer.*'
11
+
12
+ on NewLog, :create
13
+
14
+ cattr_accessor :monitor
15
+
16
+ after_setup do |rabbit_connection|
17
+ self.monitor = StatusMonitor.new(rabbit_connection)
18
+ self.monitor.start
19
+ end
20
+
21
+ def create
22
+ parser = LogParser.new(message.path)
23
+ notifier = EventNotifier.new(self.rabbit_connection)
24
+ parser.each do |event|
25
+ severity = event[:severity]
26
+ notifier.notify event if EventNotifier.significant?(severity)
27
+ self.monitor.note severity
28
+ end
29
+ notifier.wait_for_confirms
30
+ end
31
+ end
32
+
33
+ class LogParser
34
+ def initialize(path)
35
+ @file = File.open(path, 'rb')
36
+ end
37
+
38
+ def each
39
+ @file.each_line do |line|
40
+ event = parse(line)
41
+ yield event if event
42
+ end
43
+ end
44
+
45
+ protected
46
+
47
+ SEVERITY = {
48
+ 'F' => :fatal,
49
+ 'E' => :error,
50
+ 'W' => :warn,
51
+ 'I' => :info,
52
+ 'D' => :debug
53
+ }
54
+
55
+ def parse(line)
56
+ # W, [2015-08-06T13:38:16.270700 #4140] WARN -- : Sample warn
57
+ severity = SEVERITY[line[0]]
58
+ if severity
59
+ time = Time.parse(line[4..29])
60
+ message = line[49..-1]
61
+ {
62
+ severity: severity,
63
+ timestamp: time,
64
+ message: message
65
+ }
66
+ end
67
+ end
68
+ end
69
+
70
+ class EventNotifier < Tochtli::BaseClient
71
+ SIGNIFICANT_SEVERITIES = [:fatal, :error, :warn]
72
+
73
+ def self.significant?(severity)
74
+ SIGNIFICANT_SEVERITIES.include?(severity)
75
+ end
76
+
77
+ def notify(event)
78
+ publish EventOccurred.new(event), mandatory: false
79
+ end
80
+
81
+ def update_status(status)
82
+ publish CurrentStatus.new(status), mandatory: false
83
+ end
84
+ end
85
+
86
+ class StatusMonitor
87
+ include MonitorMixin
88
+
89
+ def initialize(rabbit_connection)
90
+ super()
91
+ @notifier = EventNotifier.new(rabbit_connection)
92
+ @status = Hash.new(0)
93
+ end
94
+
95
+ def start
96
+ Thread.new(&method(:monitor))
97
+ end
98
+
99
+ def note(severity)
100
+ synchronize do
101
+ @status[severity] += 1
102
+ end
103
+ end
104
+
105
+ protected
106
+
107
+ def reset_status
108
+ synchronize do
109
+ status = @status
110
+ @status = Hash.new(0)
111
+ status
112
+ end
113
+ end
114
+
115
+ def monitor
116
+ loop do
117
+ current_status = reset_status
118
+ current_status[:timestamp] = Time.now
119
+ @notifier.update_status current_status
120
+ sleep 10
121
+ end
122
+ end
123
+ end
124
+ end
125
+
126
+ Tochtli::ControllerManager.setup
127
+ Tochtli::ControllerManager.start
128
+
129
+ trap('SIGINT') { exit }
130
+ at_exit { Tochtli::ControllerManager.stop }
131
+
132
+ puts 'Press Ctrl-C to stop worker...'
133
+ sleep
@@ -0,0 +1,177 @@
1
+ require 'tochtli/version'
2
+ require 'tochtli/engine' if defined?(::Rails)
3
+
4
+ require 'bunny'
5
+ require 'json'
6
+
7
+ require 'uber/inheritable_attr'
8
+ require 'virtus'
9
+
10
+ unless defined?(ActiveSupport)
11
+ require 'facets/module/cattr'
12
+ require 'facets/array/extract_options'
13
+ require 'facets/hash/symbolize_keys'
14
+ require 'facets/hash/except'
15
+ require 'facets/string/underscore'
16
+ require 'facets/array/extract_options'
17
+ require 'facets/string/camelcase'
18
+
19
+ class String # ActiveSupport compatibility
20
+ def camelize
21
+ split('::').map { |s| s.camelcase(true) }.join('::')
22
+ end
23
+ end
24
+ end
25
+
26
+
27
+ module Tochtli
28
+ autoload :RabbitConnection, 'tochtli/rabbit_connection'
29
+ autoload :Application, 'tochtli/application'
30
+ autoload :Middleware, 'tochtli/application'
31
+ autoload :BaseController, 'tochtli/base_controller'
32
+ autoload :BaseClient, 'tochtli/base_client'
33
+ autoload :ControllerManager, 'tochtli/controller_manager'
34
+ autoload :SimpleValidation, 'tochtli/simple_validation'
35
+ autoload :Message, 'tochtli/message'
36
+ autoload :ReplyQueue, 'tochtli/reply_queue'
37
+ autoload :RabbitClient, 'tochtli/rabbit_client'
38
+ autoload :Test, 'tochtli/test'
39
+ autoload :ActiveRecordConnectionCleaner, 'tochtli/active_record_connection_cleaner'
40
+
41
+ class MessageError < StandardError
42
+ attr_reader :tochtli_message
43
+
44
+ def initialize(error_message, tochtli_message)
45
+ super error_message
46
+ @tochtli_message = tochtli_message
47
+ end
48
+ end
49
+
50
+ class InvalidMessageError < MessageError
51
+ end
52
+
53
+ class MessageDropped < MessageError
54
+ end
55
+
56
+ class << self
57
+ # Global logger for services (default: RAILS_ROOT/log/service.log)
58
+ attr_writer :logger
59
+
60
+ # Global cache store for services (default: Rails.cache)
61
+ attr_writer :cache
62
+
63
+ # If set to true bunny log level would be set to DEBUG (by default it is WARN)
64
+ attr_accessor :debug_bunny
65
+
66
+ def logger
67
+ unless @logger
68
+ if defined?(Rails)
69
+ @logger = Logger.new(File.join(Rails.root, 'log/service.log'))
70
+ @logger.level = Rails.env.production? ? Logger::WARN : Logger::DEBUG
71
+ else
72
+ @logger = Logger.new(STDERR)
73
+ @logger.level = Logger::WARN
74
+ end
75
+ end
76
+ @logger
77
+ end
78
+
79
+ def cache
80
+ if !@cache && defined?(Rails)
81
+ @cache = Rails.cache
82
+ end
83
+ @cache
84
+ end
85
+
86
+ def application
87
+ unless @application
88
+ @application = Tochtli::Application.new
89
+ @application.use_default_middlewares
90
+ end
91
+ @application
92
+ end
93
+
94
+ # Should be invoked only once
95
+ def load_services
96
+ eager_load_service_messages
97
+ eager_load_service_controllers
98
+ end
99
+
100
+ def start_services(rabbit_config=nil, logger=nil)
101
+ ControllerManager.setup(config: rabbit_config, logger: logger)
102
+ ControllerManager.start
103
+ true
104
+ rescue
105
+ if logger
106
+ logger.error "Error during service start"
107
+ logger.error "#{$!.class}: #{$!}"
108
+ logger.error $!.backtrace.join("\n")
109
+ end
110
+ false
111
+ end
112
+
113
+ def stop_services(logger=nil)
114
+ ControllerManager.stop
115
+ true
116
+ rescue
117
+ if logger
118
+ logger.error "Error during service stop"
119
+ logger.error "#{$!.class}: #{$!}"
120
+ logger.error $!.backtrace.join("\n")
121
+ end
122
+ false
123
+ end
124
+
125
+ def services_running?
126
+ ControllerManager.running?
127
+ end
128
+
129
+ def restart_services(rabbit_config=nil, logger=nil)
130
+ ControllerManager.stop if ControllerManager.running?
131
+ ControllerManager.start(rabbit_config, logger)
132
+ true
133
+ rescue
134
+ if logger
135
+ logger.error "Error during service restart"
136
+ logger.error "#{$!.class}: #{$!}"
137
+ logger.error $!.backtrace.join("\n")
138
+ end
139
+ false
140
+ end
141
+
142
+ def eager_load_service_messages
143
+ existent_engine_paths('messages').each do |load_path|
144
+ Dir.glob("#{load_path}/**/*.rb").sort.each do |file|
145
+ require file
146
+ end
147
+ end
148
+ end
149
+
150
+ def eager_load_service_controllers
151
+ existent_engine_paths('controllers').each do |load_path|
152
+ Dir.glob("#{load_path}/**/*.rb").sort.each do |file|
153
+ require file
154
+ end
155
+ end
156
+ end
157
+
158
+ def existent_engine_paths(type)
159
+ engines = ::Rails::Engine.subclasses.map(&:instance)
160
+ engines += [Rails.application]
161
+ engines.collect do |railtie|
162
+ railtie.paths["service/#{type}"].try(:existent)
163
+ end.compact.flatten
164
+ end
165
+ end
166
+ end
167
+
168
+
169
+ ####
170
+ # TEMPORARY see: https://github.com/apotonick/uber/pull/10
171
+ ####
172
+
173
+ class Uber::InheritableAttr::Clone
174
+ def self.uncloneable
175
+ [Symbol, TrueClass, FalseClass, NilClass, Numeric]
176
+ end
177
+ end
@@ -0,0 +1,9 @@
1
+ module Tochtli
2
+ class ActiveRecordConnectionCleaner < Middleware
3
+ def call(env)
4
+ @app.call(env)
5
+ ensure
6
+ ActiveRecord::Base.clear_active_connections!
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,135 @@
1
+ module Tochtli
2
+ class Application
3
+ def initialize
4
+ @middleware_stack = MiddlewareStack.new
5
+ end
6
+
7
+ def use_default_middlewares
8
+ @middleware_stack.use ErrorHandler
9
+ @middleware_stack.use MessageSetup
10
+ @middleware_stack.use MessageLogger
11
+ end
12
+
13
+ def to_app(app=nil)
14
+ app ||= MessageHandler.new
15
+ @middleware_stack.build(app)
16
+ end
17
+
18
+ def middlewares
19
+ @middleware_stack
20
+ end
21
+ end
22
+
23
+ class MiddlewareStack
24
+ def initialize
25
+ @middlewares = []
26
+ end
27
+
28
+ def use(middleware)
29
+ @middlewares << middleware
30
+ end
31
+
32
+ def build(app)
33
+ @middlewares.reverse.inject(app) { |a, e| e.new(a) }
34
+ end
35
+ end
36
+
37
+ class Middleware
38
+ def initialize(app)
39
+ @app = app
40
+ end
41
+ end
42
+
43
+ class ErrorHandler < Middleware
44
+ def call(env)
45
+ @app.call(env)
46
+ rescue Exception => ex
47
+ properties = env[:properties] || {}
48
+ controller = env[:controller]
49
+ logger = env[:logger]
50
+
51
+ logger.error "\n#{ex.class.name} (#{ex.message})"
52
+ logger.error ex.backtrace.join("\n")
53
+ if controller && properties[:reply_to]
54
+ begin
55
+ controller.reply ErrorMessage.new(error: ex.class.name, message: ex.message), properties[:reply_to], properties[:message_id]
56
+ rescue
57
+ logger.error "Unable to send error message: #{$!}"
58
+ logger.error $!.backtrace.join("\n")
59
+ end
60
+ end
61
+ false
62
+ end
63
+ end
64
+
65
+ class MessageSetup < Middleware
66
+ def call(env)
67
+ controller_class = env[:controller_class]
68
+ delivery_info = env[:delivery_info]
69
+ payload = env[:payload]
70
+ properties = env[:properties]
71
+ rabbit_connection = env[:rabbit_connection]
72
+ cache = env[:cache]
73
+ logger = env[:logger]
74
+ message_route = controller_class.find_message_route(delivery_info.routing_key)
75
+
76
+ if message_route
77
+ env[:message] = create_message(message_route.message_class, properties, payload)
78
+ env[:controller] = controller_class.new(rabbit_connection, cache, logger)
79
+ env[:action] = message_route.action
80
+
81
+ @app.call(env)
82
+
83
+ else
84
+ logger.error "\n\nMessage DROPPED by #{controller_class.name} at #{Time.now}"
85
+ logger.error "\tProperties: #{properties.inspect}."
86
+ logger.error "\tDelivery info: exchange: #{delivery_info[:exchange]}, routing_key: #{delivery_info[:routing_key]}."
87
+ false
88
+ end
89
+ end
90
+
91
+ def create_message(message_class, properties, payload)
92
+ message = message_class.new(nil, properties)
93
+ message.attributes = JSON.parse(payload)
94
+ raise InvalidMessageError.new(message.errors.join(", "), message) if message.invalid?
95
+ message
96
+ end
97
+ end
98
+
99
+ class MessageLogger < Middleware
100
+ def call(env)
101
+ start_time = Time.now
102
+ message = env[:message]
103
+ controller_class = env[:controller_class]
104
+ delivery_info = env[:delivery_info]
105
+ properties = env[:properties]
106
+ action = env[:action]
107
+ logger = env[:logger]
108
+
109
+ action_info = action.is_a?(Proc) ? "block at #{action.source_location.join(':')}" : action
110
+
111
+ logger.debug "\n\nAMQP Message #{message.class.name} at #{start_time}"
112
+ logger.debug "Processing by #{controller_class.name}##{action_info} [Thread: #{Thread.current.object_id}]"
113
+ logger.debug "\tMessage: #{message.attributes.inspect}."
114
+ logger.debug "\tProperties: #{properties.inspect}."
115
+ logger.debug "\tDelivery info: exchange: #{delivery_info[:exchange]}, routing_key: #{delivery_info[:routing_key]}."
116
+
117
+ result = @app.call(env)
118
+
119
+ logger.debug "Message #{properties[:message_id]} processed in %.1fms." % [(Time.now - start_time) * 1000]
120
+
121
+ result
122
+ end
123
+ end
124
+
125
+ class MessageHandler
126
+ def call(env)
127
+ controller = env[:controller]
128
+
129
+ controller.process_message(env)
130
+
131
+ true
132
+ end
133
+ end
134
+
135
+ end