tochtli 0.5.0

data/examples/02-log-analyzer/server.rb

@@ -0,0 +1,133 @@
+require_relative 'common'
+
+Thread.abort_on_exception = true
+
+Tochtli.logger.progname = 'SERVER'
+
+module LogAnalyzer
+
+  class LogController < Tochtli::BaseController
+    bind 'log.analyzer.*'
+
+    on NewLog, :create
+
+    cattr_accessor :monitor
+
+    after_setup do |rabbit_connection|
+	    self.monitor = StatusMonitor.new(rabbit_connection)
+	    self.monitor.start
+    end
+
+    def create
+      parser   = LogParser.new(message.path)
+      notifier = EventNotifier.new(self.rabbit_connection)
+      parser.each do |event|
+        severity = event[:severity]
+        notifier.notify event if EventNotifier.significant?(severity)
+        self.monitor.note severity
+      end
+      notifier.wait_for_confirms
+    end
+  end
+
+  class LogParser
+    def initialize(path)
+      @file = File.open(path, 'rb')
+    end
+
+    def each
+      @file.each_line do |line|
+        event = parse(line)
+        yield event if event
+      end
+    end
+
+    protected
+
+    SEVERITY = {
+            'F' => :fatal,
+            'E' => :error,
+            'W' => :warn,
+            'I' => :info,
+            'D' => :debug
+    }
+
+    def parse(line)
+      # W, [2015-08-06T13:38:16.270700 #4140]  WARN -- : Sample warn
+      severity = SEVERITY[line[0]]
+      if severity
+        time = Time.parse(line[4..29])
+        message = line[49..-1]
+        {
+                severity: severity,
+                timestamp: time,
+                message: message
+        }
+      end
+    end
+  end
+
+  class EventNotifier < Tochtli::BaseClient
+		SIGNIFICANT_SEVERITIES = [:fatal, :error, :warn]
+
+		def self.significant?(severity)
+			SIGNIFICANT_SEVERITIES.include?(severity)
+		end
+
+    def notify(event)
+      publish EventOccurred.new(event), mandatory: false
+    end
+
+    def update_status(status)
+      publish CurrentStatus.new(status), mandatory: false
+    end
+  end
+
+  class StatusMonitor
+    include MonitorMixin
+
+    def initialize(rabbit_connection)
+      super()
+      @notifier = EventNotifier.new(rabbit_connection)
+      @status = Hash.new(0)
+    end
+
+    def start
+      Thread.new(&method(:monitor))
+    end
+
+    def note(severity)
+      synchronize do
+        @status[severity] += 1
+      end
+    end
+
+    protected
+
+    def reset_status
+	    synchronize do
+		    status = @status
+		    @status = Hash.new(0)
+		    status
+	    end
+    end
+
+    def monitor
+      loop do
+        current_status = reset_status
+        current_status[:timestamp] = Time.now
+		    @notifier.update_status current_status
+        sleep 10
+      end
+    end
+  end
+end
+
+Tochtli::ControllerManager.setup
+Tochtli::ControllerManager.start
+
+trap('SIGINT') { exit }
+at_exit { Tochtli::ControllerManager.stop }
+
+puts 'Press Ctrl-C to stop worker...'
+sleep

data/lib/tochtli.rb

@@ -0,0 +1,177 @@
+require 'tochtli/version'
+require 'tochtli/engine' if defined?(::Rails)
+
+require 'bunny'
+require 'json'
+
+require 'uber/inheritable_attr'
+require 'virtus'
+
+unless defined?(ActiveSupport)
+  require 'facets/module/cattr'
+  require 'facets/array/extract_options'
+  require 'facets/hash/symbolize_keys'
+  require 'facets/hash/except'
+  require 'facets/string/underscore'
+  require 'facets/array/extract_options'
+  require 'facets/string/camelcase'
+
+  class String # ActiveSupport compatibility
+    def camelize
+      split('::').map { |s| s.camelcase(true) }.join('::')
+    end
+  end
+end
+
+
+module Tochtli
+  autoload :RabbitConnection, 'tochtli/rabbit_connection'
+  autoload :Application, 'tochtli/application'
+  autoload :Middleware, 'tochtli/application'
+  autoload :BaseController, 'tochtli/base_controller'
+  autoload :BaseClient, 'tochtli/base_client'
+  autoload :ControllerManager, 'tochtli/controller_manager'
+  autoload :SimpleValidation, 'tochtli/simple_validation'
+  autoload :Message, 'tochtli/message'
+  autoload :ReplyQueue, 'tochtli/reply_queue'
+  autoload :RabbitClient, 'tochtli/rabbit_client'
+  autoload :Test, 'tochtli/test'
+  autoload :ActiveRecordConnectionCleaner, 'tochtli/active_record_connection_cleaner'
+
+  class MessageError < StandardError
+    attr_reader :tochtli_message
+
+    def initialize(error_message, tochtli_message)
+      super error_message
+      @tochtli_message = tochtli_message
+    end
+  end
+
+  class InvalidMessageError < MessageError
+  end
+
+  class MessageDropped < MessageError
+  end
+
+  class << self
+    # Global logger for services (default: RAILS_ROOT/log/service.log)
+    attr_writer :logger
+
+    # Global cache store for services (default: Rails.cache)
+    attr_writer :cache
+
+    # If set to true bunny log level would be set to DEBUG (by default it is WARN)
+    attr_accessor :debug_bunny
+
+    def logger
+      unless @logger
+        if defined?(Rails)
+          @logger       = Logger.new(File.join(Rails.root, 'log/service.log'))
+          @logger.level = Rails.env.production? ? Logger::WARN : Logger::DEBUG
+        else
+	        @logger       = Logger.new(STDERR)
+	        @logger.level = Logger::WARN
+				end
+      end
+      @logger
+    end
+
+    def cache
+      if !@cache && defined?(Rails)
+        @cache = Rails.cache
+      end
+      @cache
+    end
+
+    def application
+      unless @application
+        @application = Tochtli::Application.new
+        @application.use_default_middlewares
+      end
+      @application
+    end
+
+    # Should be invoked only once
+    def load_services
+      eager_load_service_messages
+      eager_load_service_controllers
+    end
+
+    def start_services(rabbit_config=nil, logger=nil)
+      ControllerManager.setup(config: rabbit_config, logger: logger)
+      ControllerManager.start
+      true
+    rescue
+      if logger
+        logger.error "Error during service start"
+        logger.error "#{$!.class}: #{$!}"
+        logger.error $!.backtrace.join("\n")
+      end
+      false
+    end
+
+    def stop_services(logger=nil)
+      ControllerManager.stop
+      true
+    rescue
+      if logger
+        logger.error "Error during service stop"
+        logger.error "#{$!.class}: #{$!}"
+        logger.error $!.backtrace.join("\n")
+      end
+      false
+    end
+
+    def services_running?
+      ControllerManager.running?
+    end
+
+    def restart_services(rabbit_config=nil, logger=nil)
+      ControllerManager.stop if ControllerManager.running?
+      ControllerManager.start(rabbit_config, logger)
+      true
+    rescue
+      if logger
+        logger.error "Error during service restart"
+        logger.error "#{$!.class}: #{$!}"
+        logger.error $!.backtrace.join("\n")
+      end
+      false
+    end
+
+    def eager_load_service_messages
+      existent_engine_paths('messages').each do |load_path|
+        Dir.glob("#{load_path}/**/*.rb").sort.each do |file|
+          require file
+        end
+      end
+    end
+
+    def eager_load_service_controllers
+      existent_engine_paths('controllers').each do |load_path|
+        Dir.glob("#{load_path}/**/*.rb").sort.each do |file|
+          require file
+        end
+      end
+    end
+
+    def existent_engine_paths(type)
+      engines = ::Rails::Engine.subclasses.map(&:instance)
+      engines += [Rails.application]
+      engines.collect do |railtie|
+        railtie.paths["service/#{type}"].try(:existent)
+      end.compact.flatten
+    end
+  end
+end
+
+
+####
+# TEMPORARY see: https://github.com/apotonick/uber/pull/10
+####
+
+class Uber::InheritableAttr::Clone
+  def self.uncloneable
+    [Symbol, TrueClass, FalseClass, NilClass, Numeric]
+  end
+end

data/lib/tochtli/active_record_connection_cleaner.rb

@@ -0,0 +1,9 @@
+module Tochtli
+  class ActiveRecordConnectionCleaner < Middleware
+    def call(env)
+      @app.call(env)
+    ensure
+      ActiveRecord::Base.clear_active_connections!
+    end
+  end
+end

data/lib/tochtli/application.rb

@@ -0,0 +1,135 @@
+module Tochtli
+  class Application
+    def initialize
+      @middleware_stack = MiddlewareStack.new
+    end
+
+    def use_default_middlewares
+      @middleware_stack.use ErrorHandler
+      @middleware_stack.use MessageSetup
+      @middleware_stack.use MessageLogger
+    end
+
+    def to_app(app=nil)
+      app ||= MessageHandler.new
+      @middleware_stack.build(app)
+    end
+
+    def middlewares
+      @middleware_stack
+    end
+  end
+
+  class MiddlewareStack
+    def initialize
+      @middlewares = []
+    end
+
+    def use(middleware)
+      @middlewares << middleware
+    end
+
+    def build(app)
+      @middlewares.reverse.inject(app) { |a, e| e.new(a) }
+    end
+  end
+
+  class Middleware
+    def initialize(app)
+      @app = app
+    end
+  end
+
+  class ErrorHandler < Middleware
+    def call(env)
+      @app.call(env)
+    rescue Exception => ex
+      properties = env[:properties] || {}
+      controller = env[:controller]
+      logger     = env[:logger]
+
+      logger.error "\n#{ex.class.name} (#{ex.message})"
+      logger.error ex.backtrace.join("\n")
+      if controller && properties[:reply_to]
+        begin
+          controller.reply ErrorMessage.new(error: ex.class.name, message: ex.message), properties[:reply_to], properties[:message_id]
+        rescue
+          logger.error "Unable to send error message: #{$!}"
+          logger.error $!.backtrace.join("\n")
+        end
+      end
+      false
+    end
+  end
+
+  class MessageSetup < Middleware
+    def call(env)
+      controller_class  = env[:controller_class]
+      delivery_info     = env[:delivery_info]
+      payload           = env[:payload]
+      properties        = env[:properties]
+      rabbit_connection = env[:rabbit_connection]
+      cache             = env[:cache]
+      logger            = env[:logger]
+      message_route     = controller_class.find_message_route(delivery_info.routing_key)
+
+      if message_route
+        env[:message]    = create_message(message_route.message_class, properties, payload)
+        env[:controller] = controller_class.new(rabbit_connection, cache, logger)
+        env[:action]     = message_route.action
+
+        @app.call(env)
+
+      else
+        logger.error "\n\nMessage DROPPED by #{controller_class.name} at #{Time.now}"
+        logger.error "\tProperties: #{properties.inspect}."
+        logger.error "\tDelivery info: exchange: #{delivery_info[:exchange]}, routing_key: #{delivery_info[:routing_key]}."
+        false
+      end
+    end
+
+    def create_message(message_class, properties, payload)
+      message            = message_class.new(nil, properties)
+      message.attributes = JSON.parse(payload)
+      raise InvalidMessageError.new(message.errors.join(", "), message) if message.invalid?
+      message
+    end
+  end
+
+  class MessageLogger < Middleware
+    def call(env)
+      start_time       = Time.now
+      message          = env[:message]
+      controller_class = env[:controller_class]
+      delivery_info    = env[:delivery_info]
+      properties       = env[:properties]
+      action           = env[:action]
+      logger           = env[:logger]
+
+      action_info = action.is_a?(Proc) ? "block at #{action.source_location.join(':')}" : action
+
+      logger.debug "\n\nAMQP Message #{message.class.name} at #{start_time}"
+      logger.debug "Processing by #{controller_class.name}##{action_info} [Thread: #{Thread.current.object_id}]"
+      logger.debug "\tMessage: #{message.attributes.inspect}."
+      logger.debug "\tProperties: #{properties.inspect}."
+      logger.debug "\tDelivery info: exchange: #{delivery_info[:exchange]}, routing_key: #{delivery_info[:routing_key]}."
+
+      result = @app.call(env)
+
+      logger.debug "Message #{properties[:message_id]} processed in %.1fms." % [(Time.now - start_time) * 1000]
+
+      result
+    end
+  end
+
+  class MessageHandler
+    def call(env)
+      controller = env[:controller]
+
+      controller.process_message(env)
+
+      true
+    end
+  end
+
+end