wvanbergen-request-log-analyzer 0.2.2 → 0.3.0

data/lib/request_log_analyzer/aggregator/base.rb

@@ -0,0 +1,34 @@
+module RequestLogAnalyzer::Aggregator
+
+
+  class Base
+    
+    include RequestLogAnalyzer::FileFormat
+    
+    attr_reader :options
+    
+    def initialize(format, options = {})
+      self.register_file_format(format)
+      @options = options
+    end
+    
+    def aggregate(request)
+      # implement me!
+    end
+    
+    
+    def prepare
+    end
+    
+    def finalize
+    end
+    
+    def warning(type, message, lineno)
+    end    
+
+    def report(color = false)
+      
+    end
+
+  end
+end

data/lib/request_log_analyzer/aggregator/database.rb

@@ -0,0 +1,86 @@
+require 'rubygems'
+require 'activerecord'
+
+module RequestLogAnalyzer::Aggregator
+
+  class Database < Base
+
+    attr_reader :request_id
+
+    def prepare
+      ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => options[:database])
+
+      File.unlink(options[:database]) if File.exist?(options[:database])
+      create_database_schema!
+      
+      @request_id = 0
+    end
+    
+    def aggregate(request)
+      @request_id += 1
+      
+      request.lines.each do |line|
+        class_name = "#{line[:line_type]}_line".camelize #split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')
+        
+        attributes = line.reject { |k, v| [:line_type].include?(k) }
+        attributes[:request_id] = @request_id if options[:combined_requests]  
+        file_format.const_get(class_name).create!(attributes)
+      end
+      
+    end
+    
+    def warning(type, message, lineno)
+      file_format::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
+    end
+    
+    protected 
+    
+    def create_database_table(name, definition)
+      ActiveRecord::Migration.suppress_messages do
+        ActiveRecord::Migration.create_table("#{name}_lines") do |t|
+          t.column(:request_id, :integer) #if options[:combined_requests]
+          t.column(:lineno, :integer)
+          definition.captures.each do |field|
+            # there is only on key/value pait in this hash
+            field.each { |key, capture_type| t.column(key, column_type(capture_type)) }
+          end
+        end
+      end
+    end
+
+    def create_warning_table_and_class
+      ActiveRecord::Migration.suppress_messages do
+        ActiveRecord::Migration.create_table("warnings") do |t|
+          t.string  :warning_type, :limit => 30, :null => false
+          t.string  :message
+          t.integer :lineno          
+        end
+      end    
+      
+      file_format.const_set('Warning', Class.new(ActiveRecord::Base)) unless file_format.const_defined?('Warning')
+    end
+
+    def create_activerecord_class(name, definition)
+      class_name = "#{name}_line".camelize
+      file_format.const_set(class_name, Class.new(ActiveRecord::Base)) unless file_format.const_defined?(class_name)
+    end
+
+    def create_database_schema!
+      file_format.line_definitions.each do |name, definition|
+        create_database_table(name, definition)
+        create_activerecord_class(name, definition)
+      end
+      
+      create_warning_table_and_class
+    end
+    
+    def column_type(capture_type)
+      case capture_type
+      when :sec;   :double
+      when :msec;  :double
+      when :float; :double
+      else capture_type
+      end
+    end
+  end
+end

data/lib/request_log_analyzer/aggregator/echo.rb

@@ -0,0 +1,10 @@
+module RequestLogAnalyzer::Aggregator
+
+  class Echo < Base
+    
+    def aggregate(request)
+      puts "\nRequest: " + request.inspect
+    end
+
+  end
+end

data/lib/request_log_analyzer/aggregator/summarizer.rb

@@ -0,0 +1,53 @@
+module RequestLogAnalyzer::Aggregator
+
+  class Summarizer < Base
+    
+    attr_reader :buckets
+    
+    def prepare
+      @buckets = {}
+    end
+    
+    def aggregate(request)
+      if options[:combined_requests]
+        current_bucket_hash = @buckets
+      else       
+        @buckets[request.line_type] ||= {}
+        current_bucket_hash = @buckets[request.line_type]
+      end
+      
+      bucket_name = bucket_for(request)
+      current_bucket_hash[bucket_name] ||= default_bucket_content
+      update_bucket(current_bucket_hash[bucket_name], request)
+    end
+    
+    def default_bucket_content
+      return { :count => 0 }
+    end
+    
+    def update_bucket(bucket, request)
+      bucket[:count] += 1
+    end    
+    
+    def bucket_for(request)
+      'all'
+    end
+       
+    def report(color = false)
+      if options[:combined_requests]
+        @buckets.each do |hash, values|
+          puts "  #{hash[0..40].ljust(41)}: #{values[:count]}"
+        end     
+      else
+        @buckets.each do |line_type, buckets|
+          puts "Line type #{line_type.inspect}:"
+          buckets.each do |hash, values|
+            puts "  #{hash[0..40].ljust(41)}: #{values[:count]}"
+          end
+        end
+      end
+    end
+    
+
+  end
+end

data/lib/request_log_analyzer/controller.rb

@@ -0,0 +1,90 @@
+module RequestLogAnalyzer
+  
+  class Controller
+
+    include RequestLogAnalyzer::FileFormat
+    
+    attr_reader :aggregators
+    attr_reader :log_parser
+    attr_reader :sources
+    attr_reader :options
+
+    def self.build(arguments)
+
+      options = {}
+      options[:combined_requests] = arguments[:combined_requests]
+      options[:database] = arguments[:database] if arguments[:database]
+
+      # Create the controller with the correct file format
+      controller = Controller.new(arguments[:format].to_sym, options)
+
+      # register sources
+      arguments.files.each do |file|
+        controller << file if File.exist?(file)
+      end
+
+      # register aggregators
+      arguments[:aggregator].each { |agg| controller >> agg.to_sym } 
+
+
+      # register the database 
+      controller >> :database   if arguments[:database] && !arguments[:aggregator].include?('database')
+      controller >> :summarizer if arguments[:aggregator].empty?
+    
+      # register the echo aggregator in debug mode
+      controller >> :echo if arguments[:debug]
+            
+      return controller
+    end
+
+    def initialize(format = :rails, options = {})
+
+      @options = options
+      @aggregators = []
+      @sources     = []
+      
+      register_file_format(format)  
+      @log_parser  = RequestLogAnalyzer::LogParser.new(file_format, @options)
+      
+      @log_parser.on_warning do |type, message, lineno|        
+        @aggregators.each { |agg| agg.warning(type, message, lineno) }
+        puts "WARNING #{type.inspect} on line #{lineno}: #{message}" unless options[:silent]
+      end
+    end
+    
+    def add_aggregator(agg)
+      if agg.kind_of?(Symbol)
+        require File.dirname(__FILE__) + "/aggregator/#{agg}"
+        agg = RequestLogAnalyzer::Aggregator.const_get(agg.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join(''))
+      end
+      
+      @aggregators << agg.new(file_format, @options)
+    end
+    
+    alias :>> :add_aggregator
+    
+    def add_source(source)
+      @sources << source
+    end
+    
+    alias :<< :add_source
+    
+    def run!
+      
+      @aggregators.each { |agg| agg.prepare }
+      
+      handle_request = Proc.new { |request| @aggregators.each { |agg| agg.aggregate(request) } }
+      @sources.each do |source|
+        case source
+        when IO;     @log_parser.parse_io(source, options,   &handle_request) 
+        when String; @log_parser.parse_file(source, options, &handle_request) 
+        else;        raise "Unknwon source provided"
+        end
+      end
+      
+      @aggregators.each { |agg| agg.finalize }
+      @aggregators.each { |agg| agg.report(options[:colorize]) }
+    end
+    
+  end
+end

data/lib/request_log_analyzer/file_format/merb.rb

@@ -0,0 +1,30 @@
+module RequestLogAnalyzer::FileFormat::Merb
+
+  LINE_DEFINITIONS = {
+    
+    # ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
+    :started => {
+      :header   => true,
+      :teaser   => /Started/,
+      :regexp   => /Started request handling\:\ (.+)/,
+      :captures => [{:timestamp => :timestamp}]
+    },
+    
+    # ~ Params: {"action"=>"create", "controller"=>"session"}
+    # ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
+    :params => {
+      :teaser   => /Params/,
+      :regexp   => /Params\:\ \{(.+)\}/,
+      :captures => [{:raw_hash => :string}]
+    },
+    
+    # ~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
+    :completed => {
+      :footer   => true,
+      :teaser   => /\{:dispatch_time/,
+      :regexp   => /\{\:dispatch_time=>(\d+\.\d+(?:e-?\d+)?), (?:\:after_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?(?:\:before_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?\:action_time=>(\d+\.\d+(?:e-?\d+)?)\}/,
+      :captures => [ {:dispatch_time => :sec}, {:after_filters_time => :sec}, {:before_filters_time => :sec}, {:action_time => :sec} ]
+    }
+  }
+    
+end

data/lib/request_log_analyzer/file_format/rails.rb

@@ -0,0 +1,84 @@
+module RequestLogAnalyzer::FileFormat::Rails
+
+  # Rails < 2.1 completed line example
+  # Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
+  RAILS_21_COMPLETED = /Completed in (\d+\.\d{5}) \(\d+ reqs\/sec\) (?:\| Rendering: (\d+\.\d{5}) \(\d+\%\) )?(?:\| DB: (\d+\.\d{5}) \(\d+\%\) )?\| (\d\d\d).+\[(http.+)\]/
+
+  # Rails > 2.1 completed line example
+  # Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]  
+  RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+), )?DB: (\d+)\) \| (\d\d\d).+\[(http.+)\]/
+
+
+  LINE_DEFINITIONS = {
+    
+    # Processing EmployeeController#index (for 123.123.123.123 at 2008-07-13 06:00:00) [GET]
+    :started => {
+      :header => true,
+      :teaser => /Processing/,
+      :regexp => /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/,
+      :captures => [{:controller => :string}, {:action => :string}, {:format => :string}, {:ip => :string}, {:timestamp => :timestamp}, {:method => :string}]
+    },
+
+    # Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a998a2ff0 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a998af660@/home/floorplanner/beta/releases/20081224113708/app/controllers/page_controller.rb:14>}>] rendered_or_redirected.
+    # Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>] rendered_or_redirected.
+    :cache_hit => {
+      :regexp   => /Filter chain halted as \[\#<ActionController::Caching::Actions::ActionCacheFilter:.+>\] rendered_or_redirected/,
+      :captures => []
+    },
+    
+    # RuntimeError (Cannot destroy employee):  /app/models/employee.rb:198:in `before_destroy' 
+    :failed => {
+      :footer => true,   
+      :teaser => /Error/,
+      :regexp => /(\w+)(?:Error|Invalid) \((.*)\)\:(.*)/,
+      :captures => [{:error => :string}, {:exception_string => :string}, {:stack_trace => :string}]
+    },
+
+    # Completed lines: see above
+    :completed => {
+      :footer   => true,
+      :teaser   => /Completed in /,
+      :regexp   => Regexp.new("(?:#{RAILS_21_COMPLETED}|#{RAILS_22_COMPLETED})"),
+      :captures => [{:duration => :sec},  {:view => :sec},  {:db => :sec},  {:status => :integer}, {:url => :string},  # 2.1 variant 
+                    {:duration => :msec}, {:view => :msec}, {:db => :msec}, {:status => :integer}, {:url => :string}]  # 2.2 variant 
+
+    }
+  }   
+  
+  module Summarizer
+    
+    def bucket_for(request)
+      if options[:combined_requests]
+      
+        if request =~ :failed
+          "#{request[:error]} in #{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
+        else
+          format = request[:format] || 'html'
+          "#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
+        end
+      
+      else
+        case request.line_type
+        when :started   
+          format = request[:format] || 'html'
+          "#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
+        
+        when :completed
+          url = request[:url].downcase.split(/^http[s]?:\/\/[A-z0-9\.-]+/).last.split('?').first # only the relevant URL part
+          url << '/' if url[-1] != '/'[0] && url.length > 1 # pad a trailing slash for consistency
+
+          url.gsub!(/\/\d+-\d+-\d+(\/|$)/, '/:date') # Combine all (year-month-day) queries
+          url.gsub!(/\/\d+-\d+(\/|$)/, '/:month') # Combine all date (year-month) queries
+          url.gsub!(/\/\d+[\w-]*/, '/:id') # replace identifiers in URLs request[:url] # TODO: improve me
+          url
+        
+        when :failed
+          request[:error]
+        else
+          raise "Cannot group this request: #{request.inspect}" 
+        end
+      end
+    end
+  end
+
+end

data/lib/request_log_analyzer/file_format.rb

@@ -0,0 +1,91 @@
+module RequestLogAnalyzer
+  module FileFormat
+
+    def self.included(base)
+      base.send(:attr_reader, :file_format)
+    end
+
+    # Registers the correct language in the calling class (LogParser, Summarizer)
+    def register_file_format(format_module)
+
+      # Loads the module constant for built in file formats
+      if format_module.kind_of?(Symbol)
+        require "#{File.dirname(__FILE__)}/file_format/#{format_module}"
+        format_module = RequestLogAnalyzer::FileFormat.const_get(format_module.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')) 
+      end
+      
+      format_module.instance_eval do
+        def line_definitions
+          @line_definitions ||= self::LINE_DEFINITIONS.inject({}) do |hash, (name, definition)| 
+            hash.merge!(name => LineDefinition.new(name, definition))
+          end
+        end
+        
+        # checks whether the file format descriptor is valid
+        def valid?
+          @line_definitions.detect { |(name, ld)| ld.header } && @line_definitions.detect { |(name, ld)| ld.footer }
+        end
+      end
+      
+      # register language specific hooks in base class
+      hook_module = self.class.to_s.split('::').last
+      if format_module.const_defined?(hook_module) && format_module.const_get(hook_module).kind_of?(Module)
+        metaclass = (class << self; self; end)
+        metaclass.send(:include, format_module.const_get(hook_module))
+      end
+      
+      @file_format = format_module
+    end
+    
+    
+    
+    class LineDefinition
+
+      attr_reader :name
+      attr_accessor :teaser, :regexp, :captures
+      attr_accessor :header, :footer
+      
+      def initialize(name, definition = {})
+        @name = name
+        definition.each { |key, value| self.send("#{key.to_s}=".to_sym, value) }
+      end
+      
+      def convert_value(value, type)
+        case type
+        when :integer;   value.to_i
+        when :float;     value.to_f
+        when :decimal;   value.to_f          
+        when :symbol;    value.to_sym
+        when :sec;       value.to_f
+        when :msec;      value.to_f / 1000
+        when :timestamp; value.to_s # TODO: fix me?          
+        else value
+        end
+      end
+            
+      def matches(line, lineno = nil, parser = nil)
+        if @teaser.nil? || @teaser =~ line
+          if @regexp =~ line
+            request_info = { :line_type => name, :lineno => lineno }
+            captures_found = $~.captures
+            captures.each_with_index do |param, index|
+              unless captures_found[index].nil? || param == :ignore
+                # there is only one key/value pair in the param hash, each will only be called once
+                param.each { |key, type| request_info[key] = convert_value(captures_found[index], type) }
+              end
+            end
+            return request_info
+          else
+            parser.warn(:teaser_check_failed, "Teaser matched for #{name.inspect}, but full line did not:\n#{line.inspect}")  unless @teaser.nil? || parser.nil?
+            return false
+          end
+        else
+          return false
+        end
+      end
+      
+      alias :=~ :matches
+      
+    end
+  end
+end

data/lib/request_log_analyzer/log_parser.rb

@@ -0,0 +1,122 @@
+module RequestLogAnalyzer
+  class LogParser
+    
+    include RequestLogAnalyzer::FileFormat
+    
+    attr_reader :options
+    attr_reader :current_request
+    attr_reader :parsed_lines
+    attr_reader :parsed_requests
+    
+    def initialize(format, options = {})      
+      @line_definitions = {}
+      @options          = options
+      @parsed_lines     = 0
+      @parsed_requests  = 0
+      
+      @current_io = nil
+      
+      # install the file format module (see RequestLogAnalyzer::FileFormat)
+      # and register all the line definitions to the parser
+      self.register_file_format(format)
+    end
+              
+    def parse_files(files, options = {}, &block)
+      files.each do |file|
+        parse_file(file, options, &block)
+      end
+    end
+    
+    # Parses a file. 
+    # Creates an IO stream for the provided file, and sends it to parse_io for further handling
+    def parse_file(file, options = {}, &block)
+      @progress_handler.call(:started, file) if @progress_handler
+      File.open(file, 'r') { |f| parse_io(f, options, &block) }
+      @progress_handler.call(:completed, file) if @progress_handler
+    end    
+
+    # Finds a log line and then parses the information in the line.
+    # Yields a hash containing the information found. 
+    # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
+    # Yeilds a Hash when it encounters a chunk of information.
+    def parse_io(io, options = {}, &block)
+
+      # parse every line type by default
+      options[:line_types] ||= file_format.line_definitions.keys
+
+      # check whether all provided line types are valid
+      unknown = options[:line_types].reject { |line_type| file_format.line_definitions.has_key?(line_type) }
+      raise "Unknown line types: #{unknown.join(', ')}" unless unknown.empty?
+      
+      @current_io = io
+      @current_io.each_line do |line|
+        
+        @progress_handler.call(:progress, @current_io.pos) if @progress_handler && @current_io.lineno % 10 == 0
+        
+        request_data = nil
+        if options[:line_types].detect { |line_type| request_data = file_format.line_definitions[line_type].matches(line, @current_io.lineno, self) }
+          @parsed_lines += 1
+          if @options[:combined_requests]
+            update_current_request(request_data, &block)
+          else
+            handle_request(RequestLogAnalyzer::Request.create(@file_format, request_data), &block)
+            @parsed_requests += 1
+          end       
+        end
+      end
+      
+      warn(:unclosed_request, "End of file reached, but last request was not completed!") unless @current_request.nil?
+  
+      @current_io = nil
+    end
+    
+    # Pass a block to this function to install a progress handler
+    def on_progress(&block)
+      @progress_handler = block
+    end
+    
+    def on_warning(&block)
+      @warning_handler = block
+    end
+
+    def warn(type, message)
+      @warning_handler.call(type, message, @current_io.lineno) if @warning_handler
+    end
+    
+    protected
+    
+    def update_current_request(request_data, &block)
+      if header_line?(request_data)
+        unless @current_request.nil?
+          warn(:unclosed_request, "Encountered header line, but previous request was not closed!")
+          @current_request = nil
+        else
+          @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)              
+        end
+      else
+        unless @current_request.nil?
+          @current_request << request_data
+          if footer_line?(request_data)
+            handle_request(@current_request, &block)
+            @current_request = nil
+            @parsed_requests += 1  
+          end
+        else
+          warn(:no_current_request, "Parsebale line found outside of a request!")
+        end
+      end
+    end
+    
+    def handle_request(request, &block)
+      yield(request) if block_given?
+    end
+    
+    def header_line?(hash)
+      file_format.line_definitions[hash[:line_type]].header
+    end
+    
+    def footer_line?(hash)
+      file_format.line_definitions[hash[:line_type]].footer  
+    end 
+  end
+end

data/lib/request_log_analyzer/request.rb

@@ -0,0 +1,72 @@
+module RequestLogAnalyzer
+  class Request
+  
+    include RequestLogAnalyzer::FileFormat
+  
+    attr_reader :lines
+    
+    def initialize(file_format)
+      @lines = []
+      register_file_format(file_format)
+    end
+    
+    def self.create(file_format, *hashes)
+      request = self.new(file_format)
+      hashes.flatten.each { |hash| request << hash }
+      return request
+    end
+     
+    # Adds another line to the request
+    def << (request_info_hash)
+      @lines << request_info_hash
+    end
+    
+    # Checks whether the given line type was parsed from the log file for this request
+    def has_line_type?(line_type)
+      @lines.detect { |l| l[:line_type] == line_type.to_sym }
+    end
+    
+    alias :=~ :has_line_type?
+    
+    # Returns the value that was captured for the "field" of this request.
+    # This function will return the first value that was captured if the field
+    # was captured in multiple lines for a combined request.
+    def first(field)
+      @lines.detect { |fields| fields.has_key?(field) }[field] rescue nil
+    end
+    
+    alias :[] :first
+    
+    # Returns an array of all the "field" values that were captured for this request
+    def every(field)
+      @lines.inject([]) { |result, fields| result << fields[field] if fields.has_key?(field); result }
+    end    
+    
+    def empty?
+      @lines.length == 0
+    end
+    
+    def single_line?
+      @lines.length == 1
+    end
+    
+    def combined?
+      @lines.length > 1
+    end
+    
+    def completed?
+      header_found, footer_found = false, false
+      @lines.each do |line| 
+        line_def = file_format.line_definitions[line[:line_type]]
+        header_found = true if line_def.header
+        footer_found = true if line_def.footer        
+      end
+      header_found && footer_found
+    end
+        
+    def line_type
+      raise "Not a single line request!" unless single_line?
+      lines.first[:line_type]
+    end
+  end
+end

data/lib/request_log_analyzer.rb

@@ -0,0 +1,5 @@
+require File.dirname(__FILE__) + '/request_log_analyzer/file_format'
+require File.dirname(__FILE__) + '/request_log_analyzer/request'
+require File.dirname(__FILE__) + '/request_log_analyzer/log_parser'
+require File.dirname(__FILE__) + '/request_log_analyzer/aggregator/base'
+require File.dirname(__FILE__) + '/request_log_analyzer/controller'