request-log-analyzer 1.0.2

data/lib/request_log_analyzer/request.rb

@@ -0,0 +1,95 @@
+module RequestLogAnalyzer
+  
+  # The Request class represents a parsed request from the log file. 
+  # Instances are created by the LogParser and are passed to the different aggregators, so they
+  # can do their aggregating work. 
+  #
+  # This class provides several methods to access the data that was parsed from the log files.
+  # Request#first(field_name) returns the first (only) value corresponding to the given field
+  # Request#every(field_name) returns all values corresponding to the given field name as array.
+  class Request
+  
+    include RequestLogAnalyzer::FileFormat::Awareness
+  
+    attr_reader :lines
+    attr_reader :attributes
+        
+    # Initializes a new Request object. 
+    # It will apply the the provided FileFormat module to this instance.
+    def initialize(file_format)
+      @lines = []
+      @attributes = {}
+      register_file_format(file_format)
+    end
+    
+    # Creates a new request that was parsed from the log with the given FileFormat. The hashes
+    # that are passed to this function are added as lines to this request.
+    def self.create(file_format, *hashes)
+      request = self.new(file_format)
+      hashes.flatten.each { |hash| request << hash }
+      return request
+    end
+     
+    # Adds another line to the request.
+    # The line should be provides as a hash of the fields parsed from the line.
+    def << (request_info_hash)
+      @lines << request_info_hash
+      @attributes = request_info_hash.merge(@attributes)
+    end
+    
+    # Checks whether the given line type was parsed from the log file for this request
+    def has_line_type?(line_type)
+      return true if @lines.length == 1 && @lines[0][:line_type] == line_type.to_sym
+      
+      @lines.detect { |l| l[:line_type] == line_type.to_sym }
+    end
+    
+    alias :=~ :has_line_type?
+    
+    # Returns the value that was captured for the "field" of this request.
+    # This function will return the first value that was captured if the field
+    # was captured in multiple lines
+    def first(field)
+      @attributes[field]
+    end
+    
+    alias :[] :first
+    
+    # Returns an array of all the "field" values that were captured for this request
+    def every(field)
+      @lines.inject([]) { |result, fields| result << fields[field] if fields.has_key?(field); result }
+    end    
+    
+    # Returns true if this request does not yet contain any parsed lines. This should only occur 
+    # during parsing. An empty request should never be sent to the aggregators
+    def empty?
+      @lines.length == 0
+    end
+    
+    # Checks whether this request is completed. A completed request contains both a parsed header
+    # line and a parsed footer line. Not that calling this function in single line mode will always 
+    # return false.
+    def completed?
+      header_found, footer_found = false, false
+      @lines.each do |line| 
+        line_def = file_format.line_definitions[line[:line_type]]
+        header_found = true if line_def.header
+        footer_found = true if line_def.footer        
+      end
+      header_found && footer_found      
+    end
+    
+    # Returns the first timestamp encountered in a request. 
+    def timestamp
+      first(:timestamp)
+    end
+    
+    def first_lineno
+      @lines.first[:lineno]
+    end
+    
+    def last_lineno
+      @lines.last[:lineno]
+    end
+  end
+end

data/lib/request_log_analyzer/source/base.rb

@@ -0,0 +1,42 @@
+module RequestLogAnalyzer::Source
+  class Base
+    
+    include RequestLogAnalyzer::FileFormat::Awareness
+
+    # A hash of options
+    attr_reader :options
+
+    # The current Request object that is being parsed
+    attr_reader :current_request
+
+    # The total number of parsed lines
+    attr_reader :parsed_lines
+
+    # The total number of parsed requests.
+    attr_reader :parsed_requests
+
+    # The number of skipped lines because of warnings
+    attr_reader :skipped_lines
+
+    # Base source class used to filter input requests.
+
+    # Initializer
+    # <tt>format</tt> The file format
+    # <tt>options</tt> Are passed to the filters.
+    def initialize(format, options = {})
+      @options    = options
+      register_file_format(format)
+    end
+    
+    def prepare
+    end
+    
+    def requests(&block)
+      return true
+    end
+
+    def finalize
+    end
+
+  end
+end

data/lib/request_log_analyzer/source/log_file.rb

@@ -0,0 +1,170 @@
+module RequestLogAnalyzer::Source
+  
+  # The LogParser class reads log data from a given source and uses a file format definition
+  # to parse all relevent information about requests from the file. A FileFormat module should 
+  # be provided that contains the definitions of the lines that occur in the log data.
+  #
+  # De order in which lines occur is used to combine lines to a single request. If these lines 
+  # are mixed, requests cannot be combined properly. This can be the case if data is written to 
+  # the log file simultaneously by different mongrel processes. This problem is detected by the 
+  # parser, but the requests that are mixed up cannot be parsed. It will emit warnings when this 
+  # occurs.
+  class LogFile < RequestLogAnalyzer::Source::Base
+
+    attr_reader :source_files
+
+    # Initializes the parser instance.
+    # It will apply the language specific FileFormat module to this instance. It will use the line
+    # definitions in this module to parse any input.
+    def initialize(format, options = {})      
+      @line_definitions = {}
+      @options          = options
+      @parsed_lines     = 0
+      @parsed_requests  = 0
+      @skipped_lines    = 0
+      @current_io       = nil
+      @source_files     = options[:source_files]
+
+      # install the file format module (see RequestLogAnalyzer::FileFormat)
+      # and register all the line definitions to the parser
+      self.register_file_format(format)
+    end
+    
+    def requests(options = {}, &block)
+      
+      case @source_files
+      when IO;     
+        puts "Parsing from the standard input. Press CTRL+C to finish."
+        parse_stream(@source_files, options, &block) 
+      when String
+        parse_file(@source_files, options, &block) 
+      when Array
+        parse_files(@source_files, options, &block) 
+      else
+        raise "Unknown source provided"
+      end
+    end
+
+    # Parses a list of consequent files of the same format
+    def parse_files(files, options = {}, &block)
+      files.each { |file| parse_file(file, options, &block) }
+    end
+
+    # Parses a file. 
+    # Creates an IO stream for the provided file, and sends it to parse_io for further handling
+    def parse_file(file, options = {}, &block)
+      @progress_handler.call(:started, file) if @progress_handler
+      File.open(file, 'r') { |f| parse_io(f, options, &block) }
+      @progress_handler.call(:finished, file) if @progress_handler
+    end
+
+    def parse_stream(stream, options = {}, &block)
+      parse_io(stream, options, &block)
+    end
+
+    # Finds a log line and then parses the information in the line.
+    # Yields a hash containing the information found. 
+    # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
+    # Yeilds a Hash when it encounters a chunk of information.
+    def parse_io(io, options = {}, &block)
+
+      # parse every line type by default
+      line_types = options[:line_types] || file_format.line_definitions.keys
+
+      # check whether all provided line types are valid
+      unknown = line_types.reject { |line_type| file_format.line_definitions.has_key?(line_type) }
+      raise "Unknown line types: #{unknown.join(', ')}" unless unknown.empty?
+
+      @current_io = io
+      @current_io.each_line do |line|
+
+        @progress_handler.call(:progress, @current_io.pos) if @progress_handler && @current_io.kind_of?(File)
+
+        request_data = nil
+        line_types.each do |line_type|
+          line_type_definition = file_format.line_definitions[line_type]
+          break if request_data = line_type_definition.matches(line, @current_io.lineno, self)
+        end
+
+        if request_data
+          @parsed_lines += 1
+          update_current_request(request_data, &block)
+        end
+      end
+
+      warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
+
+      @current_io = nil
+    end
+
+    # Add a block to this method to install a progress handler while parsing
+    def progress=(proc)
+      @progress_handler = proc
+    end
+
+    # Add a block to this method to install a warning handler while parsing
+    def warning=(proc)
+      @warning_handler = proc
+    end
+
+    # This method is called by the parser if it encounteres any problems.
+    # It will call the warning handler. The default controller will pass all warnings to every
+    # aggregator that is registered and running
+    def warn(type, message)
+      @warning_handler.call(type, message, @current_io.lineno) if @warning_handler
+    end
+
+    protected
+
+    # Combines the different lines of a request into a single Request object. It will start a 
+    # new request when a header line is encountered en will emit the request when a footer line 
+    # is encountered.
+    #
+    # - Every line that is parsed before a header line is ignored as it cannot be included in 
+    #   any request. It will emit a :no_current_request warning.
+    # - A header line that is parsed before a request is closed by a footer line, is a sign of
+    #   an unprpertly ordered file. All data that is gathered for the request until then is 
+    #   discarded, the next request is ignored as well and a :unclosed_request warning is
+    #   emitted.
+    def update_current_request(request_data, &block)
+      if header_line?(request_data)
+        unless @current_request.nil?
+          if options[:assume_correct_order]
+            @parsed_requests += 1
+            yield @current_request
+            @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)
+          else
+            @skipped_lines += 1
+            warn(:unclosed_request, "Encountered header line, but previous request was not closed!")
+            @current_request = nil # remove all data that was parsed, skip next request as well.
+          end
+        else
+          @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)              
+        end
+      else
+        unless @current_request.nil?
+          @current_request << request_data
+          if footer_line?(request_data)
+            @parsed_requests += 1
+            yield @current_request
+            @current_request = nil 
+          end
+        else
+          @skipped_lines += 1
+          warn(:no_current_request, "Parsebale line found outside of a request!")
+        end
+      end
+    end
+
+    # Checks whether a given line hash is a header line.
+    def header_line?(hash)
+      file_format.line_definitions[hash[:line_type]].header
+    end
+
+    # Checks whether a given line hash is a footer line.    
+    def footer_line?(hash)
+      file_format.line_definitions[hash[:line_type]].footer  
+    end 
+  end
+
+end

data/lib/request_log_analyzer/tracker/base.rb

@@ -0,0 +1,54 @@
+module RequestLogAnalyzer
+  module Tracker
+   
+    # Base tracker. All other trackers inherit from this class
+    #
+    # Accepts the following options:
+    # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+    # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+    # * <tt>:output</tt> Direct output here (defaults to STDOUT)
+    #
+    # For example :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
+    class Base
+
+      attr_reader :options
+      
+      def initialize(options ={})
+        @options = options
+      end
+      
+      def prepare
+      end
+      
+      def update(request)
+      end
+      
+      def finalize
+      end
+      
+      def should_update?(request)
+        return false if options[:line_type] && !request.has_line_type?(options[:line_type])
+        
+        if options[:if].kind_of?(Symbol)
+          return false unless request[options[:if]]
+        elsif options[:if].respond_to?(:call)
+          return false unless options[:if].call(request)
+        end
+        
+        if options[:unless].kind_of?(Symbol)
+          return false if request[options[:unless]]
+        elsif options[:unless].respond_to?(:call)
+          return false if options[:unless].call(request)
+        end
+        
+        return true
+      end
+      
+      def report(output=STDOUT, report_width = 80, color = false)
+        output << self.inspect
+        output << "\n"  
+      end
+           
+    end 
+  end
+end

data/lib/request_log_analyzer/tracker/category.rb

@@ -0,0 +1,71 @@
+module RequestLogAnalyzer::Tracker
+
+  # Catagorize requests.
+  # Count and analyze requests for a specific attribute 
+  #
+  # Accepts the following options:
+  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+  # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+  # * <tt>:title</tt> Title do be displayed above the report.
+  # * <tt>:category</tt> Proc that handles the request categorization.
+  # * <tt>:amount</tt> The amount of lines in the report
+  #
+  # The items in the update request hash are set during the creation of the Duration tracker.
+  #
+  # Example output:
+  #  HTTP methods
+  #   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+  #  GET    ┃  22248 hits (46.2%) ┃░░░░░░░░░░░░░░░░░
+  #  PUT    ┃  13685 hits (28.4%) ┃░░░░░░░░░░░
+  #  POST   ┃  11662 hits (24.2%) ┃░░░░░░░░░
+  #  DELETE ┃    512 hits (1.1%)  ┃
+  class Category < RequestLogAnalyzer::Tracker::Base
+  
+    attr_reader :categories
+  
+    def prepare
+      raise "No categorizer set up for category tracker #{self.inspect}" unless options[:category]
+      @categories = {}
+      if options[:all_categories].kind_of?(Enumerable)
+        options[:all_categories].each { |cat| @categories[cat] = 0 }
+      end
+    end
+              
+    def update(request)
+      cat = options[:category].respond_to?(:call) ? options[:category].call(request) : request[options[:category]]
+      if !cat.nil? || options[:nils]
+        @categories[cat] ||= 0
+        @categories[cat] += 1
+      end
+    end
+  
+    def report(output = STDOUT, report_width = 80, color = false)
+      if options[:title]
+        output << "\n#{options[:title]}\n"
+        output << green(('━' * report_width), color) + "\n"
+      end
+    
+      if @categories.empty?
+        output << "None found.\n" 
+      else
+        sorted_categories = @categories.sort { |a, b| b[1] <=> a[1] }
+        total_hits        = sorted_categories.inject(0) { |carry, item| carry + item[1] }
+        sorted_categories = sorted_categories.slice(0...options[:amount]) if options[:amount]
+
+        adjuster = color ? 33 : 24 # justifcation calcultaion is slight different when color codes are inserterted
+        max_cat_length = [sorted_categories.map { |c| c[0].to_s.length }.max, report_width - adjuster].min
+        sorted_categories.each do |(cat, count)|
+          text = "%-#{max_cat_length+1}s┃%7d hits %s" % [cat.to_s[0..max_cat_length], count, (green("(%0.01f%%)", color) % [(count.to_f / total_hits) * 100])]
+          space_left  = report_width - (max_cat_length + adjuster + 3)
+          if space_left > 3
+            bar_chars  = (space_left * (count.to_f / total_hits)).round
+            output << "%-#{max_cat_length + adjuster}s %s%s" % [text, '┃', '░' * bar_chars] + "\n"
+          else
+            output << text + "\n"
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/request_log_analyzer/tracker/duration.rb

@@ -0,0 +1,81 @@
+module RequestLogAnalyzer::Tracker
+
+  # Analyze the duration of a specific attribute
+  #
+  # Accepts the following options:
+  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+  # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+  # * <tt>:title</tt> Title do be displayed above the report  
+  # * <tt>:category</tt> Proc that handles request categorization for given fileformat (REQUEST_CATEGORIZER)
+  # * <tt>:duration</tt> The field containing the duration in the request hash.
+  # * <tt>:amount</tt> The amount of lines in the report
+  #
+  # The items in the update request hash are set during the creation of the Duration tracker.
+  #
+  # Example output:
+  #  Request duration - top 20 by cumulative time   ┃    Hits ┃      Sum. |      Avg.
+  #   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+  #  EmployeeController#show.html [GET]             ┃    4742 ┃  4922.56s ┃     1.04s
+  #  EmployeeController#update.html [POST]          ┃    4647 ┃  2731.23s ┃     0.59s
+  #  EmployeeController#index.html [GET]            ┃    5802 ┃  1477.32s ┃     0.25s
+  #  .............
+  class Duration < RequestLogAnalyzer::Tracker::Base
+    attr_reader :categories
+  
+    def prepare
+      raise "No duration field set up for category tracker #{self.inspect}" unless options[:duration]
+      raise "No categorizer set up for duration tracker #{self.inspect}" unless options[:category]
+    
+      @categories = {}
+    end
+  
+    def update(request)
+      category = options[:category].respond_to?(:call) ? options[:category].call(request) : request[options[:category]]
+      duration = options[:duration].respond_to?(:call) ? options[:duration].call(request) : request[options[:duration]]
+    
+      if !duration.nil? && !category.nil?
+        @categories[category] ||= {:count => 0, :total_duration => 0.0}
+        @categories[category][:count] += 1
+        @categories[category][:total_duration] += duration
+      end
+    end
+  
+    def report_table(output = STDOUT, amount = 10, options = {}, &block)
+      
+      top_categories = @categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }.slice(0...amount)
+      max_cat_length = top_categories.map { |a| a[0].length }.max || 0
+      space_left = [options[:report_width] - 33, [max_cat_length + 1, options[:title].length].max].min
+      
+      output << "\n"
+      output << "%-#{space_left+1}s┃    Hits ┃      Sum. |      Avg." % [options[:title][0...space_left]] + "\n"
+      output << green('━' * options[:report_width], options[:color]) + "\n"
+          
+      top_categories.each do |(cat, info)|
+        hits  = info[:count]
+        total = "%0.02f" % info[:total_duration]
+        avg   = "%0.02f" % (info[:total_duration] / info[:count])
+        output << "%-#{space_left+1}s┃%8d ┃%9ss ┃%9ss" % [cat[0...space_left], hits, total, avg] + "\n"
+      end
+    end
+  
+    def report(output = STDOUT, report_width = 80, color = false)
+
+      options[:title]  ||= 'Request duration'
+      options[:report] ||= [:total, :average]
+      options[:top]    ||= 20
+    
+      options[:report].each do |report|
+        case report
+        when :average
+          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by average time", :color => color, :report_width => report_width) { |request| request[:total_duration] / request[:count] }  
+        when :total
+          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by cumulative time", :color => color, :report_width => report_width) { |request| request[:total_duration] }
+        when :hits
+          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by hits", :color => color, :report_width => report_width) { |request| request[:count] }
+        else
+          output << "Unknown duration report specified\n"
+        end
+      end
+    end      
+  end
+end