wvanbergen-request-log-analyzer 0.3.3 → 0.3.4

data/lib/request_log_analyzer/controller.rb

@@ -1,90 +0,0 @@
-module RequestLogAnalyzer
-  
-  class Controller
-
-    include RequestLogAnalyzer::FileFormat
-    
-    attr_reader :aggregators
-    attr_reader :log_parser
-    attr_reader :sources
-    attr_reader :options
-
-    def self.build(arguments)
-
-      options = {}
-      options[:combined_requests] = arguments[:combined_requests]
-      options[:database] = arguments[:database] if arguments[:database]
-
-      # Create the controller with the correct file format
-      controller = Controller.new(arguments[:format].to_sym, options)
-
-      # register sources
-      arguments.files.each do |file|
-        controller << file if File.exist?(file)
-      end
-
-      # register aggregators
-      arguments[:aggregator].each { |agg| controller >> agg.to_sym } 
-
-
-      # register the database 
-      controller >> :database   if arguments[:database] && !arguments[:aggregator].include?('database')
-      controller >> :summarizer if arguments[:aggregator].empty?
-    
-      # register the echo aggregator in debug mode
-      controller >> :echo if arguments[:debug]
-            
-      return controller
-    end
-
-    def initialize(format = :rails, options = {})
-
-      @options = options
-      @aggregators = []
-      @sources     = []
-      
-      register_file_format(format)  
-      @log_parser  = RequestLogAnalyzer::LogParser.new(file_format, @options)
-      
-      @log_parser.on_warning do |type, message, lineno|        
-        @aggregators.each { |agg| agg.warning(type, message, lineno) }
-        puts "WARNING #{type.inspect} on line #{lineno}: #{message}" unless options[:silent]
-      end
-    end
-    
-    def add_aggregator(agg)
-      if agg.kind_of?(Symbol)
-        require File.dirname(__FILE__) + "/aggregator/#{agg}"
-        agg = RequestLogAnalyzer::Aggregator.const_get(agg.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join(''))
-      end
-      
-      @aggregators << agg.new(file_format, @options)
-    end
-    
-    alias :>> :add_aggregator
-    
-    def add_source(source)
-      @sources << source
-    end
-    
-    alias :<< :add_source
-    
-    def run!
-      
-      @aggregators.each { |agg| agg.prepare }
-      
-      handle_request = Proc.new { |request| @aggregators.each { |agg| agg.aggregate(request) } }
-      @sources.each do |source|
-        case source
-        when IO;     @log_parser.parse_io(source, options,   &handle_request) 
-        when String; @log_parser.parse_file(source, options, &handle_request) 
-        else;        raise "Unknwon source provided"
-        end
-      end
-      
-      @aggregators.each { |agg| agg.finalize }
-      @aggregators.each { |agg| agg.report(options[:colorize]) }
-    end
-    
-  end
-end

data/lib/request_log_analyzer/file_format.rb

@@ -1,91 +0,0 @@
-module RequestLogAnalyzer
-  module FileFormat
-
-    def self.included(base)
-      base.send(:attr_reader, :file_format)
-    end
-
-    # Registers the correct language in the calling class (LogParser, Summarizer)
-    def register_file_format(format_module)
-
-      # Loads the module constant for built in file formats
-      if format_module.kind_of?(Symbol)
-        require "#{File.dirname(__FILE__)}/file_format/#{format_module}"
-        format_module = RequestLogAnalyzer::FileFormat.const_get(format_module.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')) 
-      end
-      
-      format_module.instance_eval do
-        def line_definitions
-          @line_definitions ||= self::LINE_DEFINITIONS.inject({}) do |hash, (name, definition)| 
-            hash.merge!(name => LineDefinition.new(name, definition))
-          end
-        end
-        
-        # checks whether the file format descriptor is valid
-        def valid?
-          @line_definitions.detect { |(name, ld)| ld.header } && @line_definitions.detect { |(name, ld)| ld.footer }
-        end
-      end
-      
-      # register language specific hooks in base class
-      hook_module = self.class.to_s.split('::').last
-      if format_module.const_defined?(hook_module) && format_module.const_get(hook_module).kind_of?(Module)
-        metaclass = (class << self; self; end)
-        metaclass.send(:include, format_module.const_get(hook_module))
-      end
-      
-      @file_format = format_module
-    end
-    
-    
-    
-    class LineDefinition
-
-      attr_reader :name
-      attr_accessor :teaser, :regexp, :captures
-      attr_accessor :header, :footer
-      
-      def initialize(name, definition = {})
-        @name = name
-        definition.each { |key, value| self.send("#{key.to_s}=".to_sym, value) }
-      end
-      
-      def convert_value(value, type)
-        case type
-        when :integer;   value.to_i
-        when :float;     value.to_f
-        when :decimal;   value.to_f          
-        when :symbol;    value.to_sym
-        when :sec;       value.to_f
-        when :msec;      value.to_f / 1000
-        when :timestamp; value.to_s # TODO: fix me?          
-        else value
-        end
-      end
-            
-      def matches(line, lineno = nil, parser = nil)
-        if @teaser.nil? || @teaser =~ line
-          if @regexp =~ line
-            request_info = { :line_type => name, :lineno => lineno }
-            captures_found = $~.captures
-            captures.each_with_index do |param, index|
-              unless captures_found[index].nil? || param == :ignore
-                # there is only one key/value pair in the param hash, each will only be called once
-                param.each { |key, type| request_info[key] = convert_value(captures_found[index], type) }
-              end
-            end
-            return request_info
-          else
-            parser.warn(:teaser_check_failed, "Teaser matched for #{name.inspect}, but full line did not:\n#{line.inspect}")  unless @teaser.nil? || parser.nil?
-            return false
-          end
-        else
-          return false
-        end
-      end
-      
-      alias :=~ :matches
-      
-    end
-  end
-end

data/lib/request_log_analyzer/file_format/merb.rb

@@ -1,30 +0,0 @@
-module RequestLogAnalyzer::FileFormat::Merb
-
-  LINE_DEFINITIONS = {
-    
-    # ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
-    :started => {
-      :header   => true,
-      :teaser   => /Started/,
-      :regexp   => /Started request handling\:\ (.+)/,
-      :captures => [{:timestamp => :timestamp}]
-    },
-    
-    # ~ Params: {"action"=>"create", "controller"=>"session"}
-    # ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
-    :params => {
-      :teaser   => /Params/,
-      :regexp   => /Params\:\ \{(.+)\}/,
-      :captures => [{:raw_hash => :string}]
-    },
-    
-    # ~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
-    :completed => {
-      :footer   => true,
-      :teaser   => /\{:dispatch_time/,
-      :regexp   => /\{\:dispatch_time=>(\d+\.\d+(?:e-?\d+)?), (?:\:after_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?(?:\:before_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?\:action_time=>(\d+\.\d+(?:e-?\d+)?)\}/,
-      :captures => [ {:dispatch_time => :sec}, {:after_filters_time => :sec}, {:before_filters_time => :sec}, {:action_time => :sec} ]
-    }
-  }
-    
-end

data/lib/request_log_analyzer/file_format/rails.rb

@@ -1,84 +0,0 @@
-module RequestLogAnalyzer::FileFormat::Rails
-
-  # Rails < 2.1 completed line example
-  # Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
-  RAILS_21_COMPLETED = /Completed in (\d+\.\d{5}) \(\d+ reqs\/sec\) (?:\| Rendering: (\d+\.\d{5}) \(\d+\%\) )?(?:\| DB: (\d+\.\d{5}) \(\d+\%\) )?\| (\d\d\d).+\[(http.+)\]/
-
-  # Rails > 2.1 completed line example
-  # Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]  
-  RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+), )?DB: (\d+)\) \| (\d\d\d).+\[(http.+)\]/
-
-
-  LINE_DEFINITIONS = {
-    
-    # Processing EmployeeController#index (for 123.123.123.123 at 2008-07-13 06:00:00) [GET]
-    :started => {
-      :header => true,
-      :teaser => /Processing/,
-      :regexp => /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/,
-      :captures => [{:controller => :string}, {:action => :string}, {:format => :string}, {:ip => :string}, {:timestamp => :timestamp}, {:method => :string}]
-    },
-
-    # Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a998a2ff0 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a998af660@/home/floorplanner/beta/releases/20081224113708/app/controllers/page_controller.rb:14>}>] rendered_or_redirected.
-    # Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>] rendered_or_redirected.
-    :cache_hit => {
-      :regexp   => /Filter chain halted as \[\#<ActionController::Caching::Actions::ActionCacheFilter:.+>\] rendered_or_redirected/,
-      :captures => []
-    },
-    
-    # RuntimeError (Cannot destroy employee):  /app/models/employee.rb:198:in `before_destroy' 
-    :failed => {
-      :footer => true,   
-      :teaser => /Error/,
-      :regexp => /(\w+)(?:Error|Invalid) \((.*)\)\:(.*)/,
-      :captures => [{:error => :string}, {:exception_string => :string}, {:stack_trace => :string}]
-    },
-
-    # Completed lines: see above
-    :completed => {
-      :footer   => true,
-      :teaser   => /Completed in /,
-      :regexp   => Regexp.new("(?:#{RAILS_21_COMPLETED}|#{RAILS_22_COMPLETED})"),
-      :captures => [{:duration => :sec},  {:view => :sec},  {:db => :sec},  {:status => :integer}, {:url => :string},  # 2.1 variant 
-                    {:duration => :msec}, {:view => :msec}, {:db => :msec}, {:status => :integer}, {:url => :string}]  # 2.2 variant 
-
-    }
-  }   
-  
-  module Summarizer
-    
-    def bucket_for(request)
-      if options[:combined_requests]
-      
-        if request =~ :failed
-          "#{request[:error]} in #{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
-        else
-          format = request[:format] || 'html'
-          "#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
-        end
-      
-      else
-        case request.line_type
-        when :started   
-          format = request[:format] || 'html'
-          "#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
-        
-        when :completed
-          url = request[:url].downcase.split(/^http[s]?:\/\/[A-z0-9\.-]+/).last.split('?').first # only the relevant URL part
-          url << '/' if url[-1] != '/'[0] && url.length > 1 # pad a trailing slash for consistency
-
-          url.gsub!(/\/\d+-\d+-\d+(\/|$)/, '/:date') # Combine all (year-month-day) queries
-          url.gsub!(/\/\d+-\d+(\/|$)/, '/:month') # Combine all date (year-month) queries
-          url.gsub!(/\/\d+[\w-]*/, '/:id') # replace identifiers in URLs request[:url] # TODO: improve me
-          url
-        
-        when :failed
-          request[:error]
-        else
-          raise "Cannot group this request: #{request.inspect}" 
-        end
-      end
-    end
-  end
-
-end

data/lib/request_log_analyzer/log_parser.rb

@@ -1,122 +0,0 @@
-module RequestLogAnalyzer
-  class LogParser
-    
-    include RequestLogAnalyzer::FileFormat
-    
-    attr_reader :options
-    attr_reader :current_request
-    attr_reader :parsed_lines
-    attr_reader :parsed_requests
-    
-    def initialize(format, options = {})      
-      @line_definitions = {}
-      @options          = options
-      @parsed_lines     = 0
-      @parsed_requests  = 0
-      
-      @current_io = nil
-      
-      # install the file format module (see RequestLogAnalyzer::FileFormat)
-      # and register all the line definitions to the parser
-      self.register_file_format(format)
-    end
-              
-    def parse_files(files, options = {}, &block)
-      files.each do |file|
-        parse_file(file, options, &block)
-      end
-    end
-    
-    # Parses a file. 
-    # Creates an IO stream for the provided file, and sends it to parse_io for further handling
-    def parse_file(file, options = {}, &block)
-      @progress_handler.call(:started, file) if @progress_handler
-      File.open(file, 'r') { |f| parse_io(f, options, &block) }
-      @progress_handler.call(:completed, file) if @progress_handler
-    end    
-
-    # Finds a log line and then parses the information in the line.
-    # Yields a hash containing the information found. 
-    # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
-    # Yeilds a Hash when it encounters a chunk of information.
-    def parse_io(io, options = {}, &block)
-
-      # parse every line type by default
-      options[:line_types] ||= file_format.line_definitions.keys
-
-      # check whether all provided line types are valid
-      unknown = options[:line_types].reject { |line_type| file_format.line_definitions.has_key?(line_type) }
-      raise "Unknown line types: #{unknown.join(', ')}" unless unknown.empty?
-      
-      @current_io = io
-      @current_io.each_line do |line|
-        
-        @progress_handler.call(:progress, @current_io.pos) if @progress_handler && @current_io.lineno % 10 == 0
-        
-        request_data = nil
-        if options[:line_types].detect { |line_type| request_data = file_format.line_definitions[line_type].matches(line, @current_io.lineno, self) }
-          @parsed_lines += 1
-          if @options[:combined_requests]
-            update_current_request(request_data, &block)
-          else
-            handle_request(RequestLogAnalyzer::Request.create(@file_format, request_data), &block)
-            @parsed_requests += 1
-          end       
-        end
-      end
-      
-      warn(:unclosed_request, "End of file reached, but last request was not completed!") unless @current_request.nil?
-  
-      @current_io = nil
-    end
-    
-    # Pass a block to this function to install a progress handler
-    def on_progress(&block)
-      @progress_handler = block
-    end
-    
-    def on_warning(&block)
-      @warning_handler = block
-    end
-
-    def warn(type, message)
-      @warning_handler.call(type, message, @current_io.lineno) if @warning_handler
-    end
-    
-    protected
-    
-    def update_current_request(request_data, &block)
-      if header_line?(request_data)
-        unless @current_request.nil?
-          warn(:unclosed_request, "Encountered header line, but previous request was not closed!")
-          @current_request = nil
-        else
-          @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)              
-        end
-      else
-        unless @current_request.nil?
-          @current_request << request_data
-          if footer_line?(request_data)
-            handle_request(@current_request, &block)
-            @current_request = nil
-            @parsed_requests += 1  
-          end
-        else
-          warn(:no_current_request, "Parsebale line found outside of a request!")
-        end
-      end
-    end
-    
-    def handle_request(request, &block)
-      yield(request) if block_given?
-    end
-    
-    def header_line?(hash)
-      file_format.line_definitions[hash[:line_type]].header
-    end
-    
-    def footer_line?(hash)
-      file_format.line_definitions[hash[:line_type]].footer  
-    end 
-  end
-end

data/lib/request_log_analyzer/request.rb

@@ -1,72 +0,0 @@
-module RequestLogAnalyzer
-  class Request
-  
-    include RequestLogAnalyzer::FileFormat
-  
-    attr_reader :lines
-    
-    def initialize(file_format)
-      @lines = []
-      register_file_format(file_format)
-    end
-    
-    def self.create(file_format, *hashes)
-      request = self.new(file_format)
-      hashes.flatten.each { |hash| request << hash }
-      return request
-    end
-     
-    # Adds another line to the request
-    def << (request_info_hash)
-      @lines << request_info_hash
-    end
-    
-    # Checks whether the given line type was parsed from the log file for this request
-    def has_line_type?(line_type)
-      @lines.detect { |l| l[:line_type] == line_type.to_sym }
-    end
-    
-    alias :=~ :has_line_type?
-    
-    # Returns the value that was captured for the "field" of this request.
-    # This function will return the first value that was captured if the field
-    # was captured in multiple lines for a combined request.
-    def first(field)
-      @lines.detect { |fields| fields.has_key?(field) }[field] rescue nil
-    end
-    
-    alias :[] :first
-    
-    # Returns an array of all the "field" values that were captured for this request
-    def every(field)
-      @lines.inject([]) { |result, fields| result << fields[field] if fields.has_key?(field); result }
-    end    
-    
-    def empty?
-      @lines.length == 0
-    end
-    
-    def single_line?
-      @lines.length == 1
-    end
-    
-    def combined?
-      @lines.length > 1
-    end
-    
-    def completed?
-      header_found, footer_found = false, false
-      @lines.each do |line| 
-        line_def = file_format.line_definitions[line[:line_type]]
-        header_found = true if line_def.header
-        footer_found = true if line_def.footer        
-      end
-      header_found && footer_found
-    end
-        
-    def line_type
-      raise "Not a single line request!" unless single_line?
-      lines.first[:line_type]
-    end
-  end
-end