wvanbergen-request-log-analyzer 1.0.0 → 1.0.1

data/lib/request_log_analyzer/source/log_file.rb

@@ -0,0 +1,180 @@
+module RequestLogAnalyzer::Source
+  # The LogParser class reads log data from a given source and uses a file format definition
+  # to parse all relevent information about requests from the file.
+  #
+  # A FileFormat module should be provided that contains the definitions of the lines that 
+  # occur in the log data. The log parser can run in two modes:
+  # - In single line mode, it will emit every detected line as a separate request
+  # - In combined requests mode, it will combine the different lines from the line defintions
+  #   into one request, that will then be emitted. 
+  #
+  # The combined requests mode gives better information, but can be problematic if the log 
+  # file is unordered. This can be the case if data is written to the log file simultaneously 
+  # by different mongrel processes. This problem is detected by the parser, but the requests
+  # that are mixed up cannot be parsed. It will emit warnings when this occurs.
+  class LogFile < RequestLogAnalyzer::Source::Base
+
+    attr_reader :source_files
+
+    # Initializes the parser instance.
+    # It will apply the language specific FileFormat module to this instance. It will use the line
+    # definitions in this module to parse any input.
+    def initialize(format, options = {})      
+      @line_definitions = {}
+      @options          = options
+      @parsed_lines     = 0
+      @parsed_requests  = 0
+      @skipped_requests = 0
+      @current_io = nil
+      @source_files     = options[:source_files]
+
+      # install the file format module (see RequestLogAnalyzer::FileFormat)
+      # and register all the line definitions to the parser
+      self.register_file_format(format)
+    end
+    
+    def requests(options = {}, &block)
+      
+      case @source_files
+      when IO;     
+        puts "Parsing from the standard input. Press CTRL+C to finish."
+        parse_stream(@source_files, options, &block) 
+      when String
+        parse_file(@source_files, options, &block) 
+      when Array
+        parse_files(@source_files, options, &block) 
+      else
+        raise "Unknown source provided"
+      end
+    end
+
+    # Parses a list of consequent files of the same format
+    def parse_files(files, options = {}, &block)
+      files.each { |file| parse_file(file, options, &block) }
+    end
+
+    # Parses a file. 
+    # Creates an IO stream for the provided file, and sends it to parse_io for further handling
+    def parse_file(file, options = {}, &block)
+      @progress_handler.call(:started, file) if @progress_handler
+      File.open(file, 'r') { |f| parse_io(f, options, &block) }
+      @progress_handler.call(:finished, file) if @progress_handler
+    end
+
+    def parse_stream(stream, options = {}, &block)
+      parse_io(stream, options, &block)
+    end
+
+    # Finds a log line and then parses the information in the line.
+    # Yields a hash containing the information found. 
+    # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
+    # Yeilds a Hash when it encounters a chunk of information.
+    def parse_io(io, options = {}, &block)
+
+      # parse every line type by default
+      line_types = options[:line_types] || file_format.line_definitions.keys
+
+      # check whether all provided line types are valid
+      unknown = line_types.reject { |line_type| file_format.line_definitions.has_key?(line_type) }
+      raise "Unknown line types: #{unknown.join(', ')}" unless unknown.empty?
+
+      puts "Parsing mode: " + (options[:combined_requests] ? 'combined requests' : 'single lines') if options[:debug]
+
+      @current_io = io
+      @current_io.each_line do |line|
+
+        @progress_handler.call(:progress, @current_io.pos) if @progress_handler && @current_io.kind_of?(File)
+
+        request_data = nil
+        line_types.each do |line_type|
+          line_type_definition = file_format.line_definitions[line_type]
+          break if request_data = line_type_definition.matches(line, @current_io.lineno, self)
+        end
+
+        if request_data
+          @parsed_lines += 1
+          if @options[:combined_requests]
+            update_current_request(request_data, &block)
+          else
+            @parsed_requests +=1
+            yield RequestLogAnalyzer::Request.create(@file_format, request_data)
+          end       
+        end
+      end
+
+      warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
+
+      @current_io = nil
+    end
+
+    # Add a block to this method to install a progress handler while parsing
+    def progress=(proc)
+      @progress_handler = proc
+    end
+
+    # Add a block to this method to install a warning handler while parsing
+    def warning=(proc)
+      @warning_handler = proc
+    end
+
+    # This method is called by the parser if it encounteres any problems.
+    # It will call the warning handler. The default controller will pass all warnings to every
+    # aggregator that is registered and running
+    def warn(type, message)
+      @warning_handler.call(type, message, @current_io.lineno) if @warning_handler
+    end
+
+    protected
+
+    # Combines the different lines of a request into a single Request object.
+    # This function is only called in combined requests mode. It will start a new request when
+    # a header line is encountered en will emit the request when a footer line is encountered.
+    #
+    # - Every line that is parsed before a header line is ignored as it cannot be included in 
+    #   any request. It will emit a :no_current_request warning.
+    # - A header line that is parsed before a request is closed by a footer line, is a sign of
+    #   an unprpertly ordered file. All data that is gathered for the request until then is 
+    #   discarded, the next request is ignored as well and a :unclosed_request warning is
+    #   emitted.
+    def update_current_request(request_data, &block)
+      if header_line?(request_data)
+        unless @current_request.nil?
+          if options[:assume_correct_order]
+            @parsed_requests += 1
+            yield @current_request
+            @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)
+          else
+            @skipped_requests += 1
+            warn(:unclosed_request, "Encountered header line, but previous request was not closed!")
+            @current_request = nil # remove all data that was parsed, skip next request as well.
+          end
+        else
+          @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)              
+        end
+      else
+        unless @current_request.nil?
+          @current_request << request_data
+          if footer_line?(request_data)
+            @parsed_requests += 1
+            yield @current_request
+            @current_request = nil 
+          end
+        else
+          @skipped_requests += 1
+          warn(:no_current_request, "Parsebale line found outside of a request!")
+        end
+      end
+    end
+
+    # Checks whether a given line hash is a header line.
+    def header_line?(hash)
+      file_format.line_definitions[hash[:line_type]].header
+    end
+
+    # Checks whether a given line hash is a footer line.    
+    def footer_line?(hash)
+      file_format.line_definitions[hash[:line_type]].footer  
+    end 
+  end
+
+end

data/lib/request_log_analyzer/tracker/base.rb

@@ -0,0 +1,54 @@
+module RequestLogAnalyzer
+  module Tracker
+   
+    # Base tracker. All other trackers inherit from this class
+    #
+    # Accepts the following options:
+    # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+    # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+    # * <tt>:output</tt> Direct output here (defaults to STDOUT)
+    #
+    # For example :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
+    class Base
+
+      attr_reader :options
+      
+      def initialize(options ={})
+        @options = options
+      end
+      
+      def prepare
+      end
+      
+      def update(request)
+      end
+      
+      def finalize
+      end
+      
+      def should_update?(request)
+        return false if options[:line_type] && !request.has_line_type?(options[:line_type])
+        
+        if options[:if].kind_of?(Symbol)
+          return false unless request[options[:if]]
+        elsif options[:if].respond_to?(:call)
+          return false unless options[:if].call(request)
+        end
+        
+        if options[:unless].kind_of?(Symbol)
+          return false if request[options[:unless]]
+        elsif options[:unless].respond_to?(:call)
+          return false if options[:unless].call(request)
+        end
+        
+        return true
+      end
+      
+      def report(output=STDOUT, report_width = 80, color = false)
+        output << self.inspect
+        output << "\n"  
+      end
+           
+    end 
+  end
+end

data/lib/request_log_analyzer/tracker/category.rb

@@ -0,0 +1,71 @@
+module RequestLogAnalyzer::Tracker
+
+  # Catagorize requests.
+  # Count and analyze requests for a specific attribute 
+  #
+  # Accepts the following options:
+  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+  # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+  # * <tt>:title</tt> Title do be displayed above the report.
+  # * <tt>:category</tt> Proc that handles the request categorization.
+  # * <tt>:amount</tt> The amount of lines in the report
+  #
+  # The items in the update request hash are set during the creation of the Duration tracker.
+  #
+  # Example output:
+  #  HTTP methods
+  #   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+  #  GET    ┃  22248 hits (46.2%) ┃░░░░░░░░░░░░░░░░░
+  #  PUT    ┃  13685 hits (28.4%) ┃░░░░░░░░░░░
+  #  POST   ┃  11662 hits (24.2%) ┃░░░░░░░░░
+  #  DELETE ┃    512 hits (1.1%)  ┃
+  class Category < RequestLogAnalyzer::Tracker::Base
+  
+    attr_reader :categories
+  
+    def prepare
+      raise "No categorizer set up for category tracker #{self.inspect}" unless options[:category]
+      @categories = {}
+      if options[:all_categories].kind_of?(Enumerable)
+        options[:all_categories].each { |cat| @categories[cat] = 0 }
+      end
+    end
+              
+    def update(request)
+      cat = options[:category].respond_to?(:call) ? options[:category].call(request) : request[options[:category]]
+      if !cat.nil? || options[:nils]
+        @categories[cat] ||= 0
+        @categories[cat] += 1
+      end
+    end
+  
+    def report(output = STDOUT, report_width = 80, color = false)
+      if options[:title]
+        output << "\n#{options[:title]}\n"
+        output << green(('━' * report_width), color) + "\n"
+      end
+    
+      if @categories.empty?
+        output << "None found.\n" 
+      else
+        sorted_categories = @categories.sort { |a, b| b[1] <=> a[1] }
+        total_hits        = sorted_categories.inject(0) { |carry, item| carry + item[1] }
+        sorted_categories = sorted_categories.slice(0...options[:amount]) if options[:amount]
+
+        adjuster = color ? 33 : 24 # justifcation calcultaion is slight different when color codes are inserterted
+        max_cat_length = [sorted_categories.map { |c| c[0].to_s.length }.max, report_width - adjuster].min
+        sorted_categories.each do |(cat, count)|
+          text = "%-#{max_cat_length+1}s┃%7d hits %s" % [cat.to_s[0..max_cat_length], count, (green("(%0.01f%%)", color) % [(count.to_f / total_hits) * 100])]
+          space_left  = report_width - (max_cat_length + adjuster + 3)
+          if space_left > 3
+            bar_chars  = (space_left * (count.to_f / total_hits)).round
+            output << "%-#{max_cat_length + adjuster}s %s%s" % [text, '┃', '░' * bar_chars] + "\n"
+          else
+            output << text + "\n"
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/request_log_analyzer/tracker/duration.rb

@@ -0,0 +1,81 @@
+module RequestLogAnalyzer::Tracker
+
+  # Analyze the duration of a specific attribute
+  #
+  # Accepts the following options:
+  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+  # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+  # * <tt>:title</tt> Title do be displayed above the report  
+  # * <tt>:category</tt> Proc that handles request categorization for given fileformat (REQUEST_CATEGORIZER)
+  # * <tt>:duration</tt> The field containing the duration in the request hash.
+  # * <tt>:amount</tt> The amount of lines in the report
+  #
+  # The items in the update request hash are set during the creation of the Duration tracker.
+  #
+  # Example output:
+  #  Request duration - top 20 by cumulative time   ┃    Hits ┃      Sum. |      Avg.
+  #   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+  #  EmployeeController#show.html [GET]             ┃    4742 ┃  4922.56s ┃     1.04s
+  #  EmployeeController#update.html [POST]          ┃    4647 ┃  2731.23s ┃     0.59s
+  #  EmployeeController#index.html [GET]            ┃    5802 ┃  1477.32s ┃     0.25s
+  #  .............
+  class Duration < RequestLogAnalyzer::Tracker::Base
+    attr_reader :categories
+  
+    def prepare
+      raise "No duration field set up for category tracker #{self.inspect}" unless options[:duration]
+      raise "No categorizer set up for duration tracker #{self.inspect}" unless options[:category]
+    
+      @categories = {}
+    end
+  
+    def update(request)
+      category = options[:category].respond_to?(:call) ? options[:category].call(request) : request[options[:category]]
+      duration = options[:duration].respond_to?(:call) ? options[:duration].call(request) : request[options[:duration]]
+    
+      if !duration.nil? && !category.nil?
+        @categories[category] ||= {:count => 0, :total_duration => 0.0}
+        @categories[category][:count] += 1
+        @categories[category][:total_duration] += duration
+      end
+    end
+  
+    def report_table(output = STDOUT, amount = 10, options = {}, &block)
+      
+      top_categories = @categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }.slice(0...amount)
+      max_cat_length = top_categories.map { |a| a[0].length }.max || 0
+      space_left = [options[:report_width] - 33, [max_cat_length + 1, options[:title].length].max].min
+      
+      output << "\n"
+      output << "%-#{space_left+1}s┃    Hits ┃      Sum. |      Avg." % [options[:title][0...space_left]] + "\n"
+      output << green('━' * options[:report_width], options[:color]) + "\n"
+          
+      top_categories.each do |(cat, info)|
+        hits  = info[:count]
+        total = "%0.02f" % info[:total_duration]
+        avg   = "%0.02f" % (info[:total_duration] / info[:count])
+        output << "%-#{space_left+1}s┃%8d ┃%9ss ┃%9ss" % [cat[0...space_left], hits, total, avg] + "\n"
+      end
+    end
+  
+    def report(output = STDOUT, report_width = 80, color = false)
+
+      options[:title]  ||= 'Request duration'
+      options[:report] ||= [:total, :average]
+      options[:top]    ||= 20
+    
+      options[:report].each do |report|
+        case report
+        when :average
+          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by average time", :color => color, :report_width => report_width) { |request| request[:total_duration] / request[:count] }  
+        when :total
+          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by cumulative time", :color => color, :report_width => report_width) { |request| request[:total_duration] }
+        when :hits
+          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by hits", :color => color, :report_width => report_width) { |request| request[:count] }
+        else
+          output << "Unknown duration report specified\n"
+        end
+      end
+    end      
+  end
+end

data/lib/request_log_analyzer/tracker/hourly_spread.rb

@@ -0,0 +1,80 @@
+module RequestLogAnalyzer::Tracker
+
+  # Determines the average hourly spread of the parsed requests.
+  # This spread is shown in a graph form.
+  #
+  # Accepts the following options:
+  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+  # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+  #
+  # Expects the following items in the update request hash
+  # * <tt>:timestamp</tt> in YYYYMMDDHHMMSS format.
+  #
+  # Example output:
+  #  Requests graph - average per day per hour
+  #   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+  #    7:00 - 330 hits        : ░░░░░░░
+  #    8:00 - 704 hits        : ░░░░░░░░░░░░░░░░░
+  #    9:00 - 830 hits        : ░░░░░░░░░░░░░░░░░░░░
+  #   10:00 - 822 hits        : ░░░░░░░░░░░░░░░░░░░
+  #   11:00 - 823 hits        : ░░░░░░░░░░░░░░░░░░░
+  #   12:00 - 729 hits        : ░░░░░░░░░░░░░░░░░
+  #   13:00 - 614 hits        : ░░░░░░░░░░░░░░
+  #   14:00 - 690 hits        : ░░░░░░░░░░░░░░░░
+  #   15:00 - 492 hits        : ░░░░░░░░░░░
+  #   16:00 - 355 hits        : ░░░░░░░░
+  #   17:00 - 213 hits        : ░░░░░
+  #   18:00 - 107 hits        : ░░
+  #   ................
+  class HourlySpread < RequestLogAnalyzer::Tracker::Base
+
+    attr_reader :first, :last, :request_time_graph
+  
+    def prepare
+      options[:field] ||= :timestamp
+      @request_time_graph = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+    end
+              
+    def update(request)
+      request = request.attributes
+      timestamp = request[options[:field]]
+
+      @request_time_graph[timestamp.to_s[8..9].to_i] +=1
+      @first = timestamp if @first.nil? || timestamp < @first
+      @last  = timestamp if @last.nil?  || timestamp > @last
+    end
+  
+    def report(output = STDOUT, report_width = 80, color = false)
+      output << "\n"
+      output << "Requests graph - average per day per hour\n"
+      output << green("━" * report_width, color) + "\n"
+      
+      if @request_time_graph == [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+        output << "None found.\n"
+        return
+      end
+
+      first_date    = DateTime.parse(@first.to_s, '%Y%m%d%H%M%S')
+      last_date     = DateTime.parse(@last.to_s, '%Y%m%d%H%M%S')
+      days          = (@last && @first) ? (last_date - first_date).ceil : 1
+      deviation     = @request_time_graph.max / 20
+      deviation     = 1 if deviation == 0
+      color_cutoff  = 15
+      
+      @request_time_graph.each_with_index do |requests, index|
+        display_chars = requests / deviation
+        request_today = requests / days
+      
+        if display_chars >= color_cutoff
+          display_chars_string = green(('░' * color_cutoff), color) + red(('░' * (display_chars - color_cutoff)), color)
+        else
+          display_chars_string = green(('░' * display_chars), color)
+        end
+      
+        output << "#{index.to_s.rjust(3)}:00 - #{(request_today.to_s + ' hits').ljust(15)} : #{display_chars_string}\n"
+      end
+      output << "\n"
+
+    end
+  end
+end

data/lib/request_log_analyzer/tracker/timespan.rb

@@ -0,0 +1,54 @@
+module RequestLogAnalyzer::Tracker
+
+  # Determines the datetime of the first request and the last request
+  # Also determines the amount of days inbetween these.
+  #
+  # Accepts the following options:
+  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+  # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+  # * <tt>:field</tt> The timestamp field that is looked at. Defaults to :timestamp.
+  # * <tt>:title</tt> Title do be displayed above the report.
+  #
+  # Expects the following items in the update request hash
+  # * <tt>:timestamp</tt> in YYYYMMDDHHMMSS format.
+  #
+  # Example output:
+  #  First request:        2008-07-13 06:25:06
+  #  Last request:         2008-07-20 06:18:06
+  #  Total time analyzed:  7 days
+  class Timespan < RequestLogAnalyzer::Tracker::Base
+
+    attr_reader :first, :last, :request_time_graph
+  
+    def prepare
+      options[:field] ||= :timestamp
+    end
+              
+    def update(request)
+      timestamp = request[options[:field]]
+
+      @first = timestamp if @first.nil? || timestamp < @first
+      @last  = timestamp if @last.nil?  || timestamp > @last
+    end
+  
+    def report(output = STDOUT, report_width = 80, color = false)
+      if options[:title]
+        output << "\n#{options[:title]}\n"
+        output << green('━' * options[:title].length, color) + "\n"
+      end
+      
+      first_date  = DateTime.parse(@first.to_s, '%Y%m%d%H%M%S') rescue nil
+      last_date   = DateTime.parse(@last.to_s, '%Y%m%d%H%M%S') rescue nil
+      
+      if @last && @first
+        days        = (@last && @first) ? (last_date - first_date).ceil : 1
+
+        output << "First request:        #{first_date.strftime('%Y-%m-%d %H:%M:%I')}\n"
+        output << "Last request:         #{last_date.strftime('%Y-%m-%d %H:%M:%I')}\n"
+        output << "Total time analyzed:  #{days} days\n"
+      end
+      output << "\n"
+      
+    end
+  end
+end

data/spec/file_format_spec.rb

@@ -0,0 +1,78 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe RequestLogAnalyzer::FileFormat, :format_definition do
+  
+  before(:each) do
+    @first_file_format = Class.new(RequestLogAnalyzer::FileFormat)
+    @second_file_format = Class.new(RequestLogAnalyzer::FileFormat)
+  end
+    
+  it "should specify lines with a hash" do
+    
+    @first_file_format.new.should have(0).line_definitions
+        
+    @first_file_format.format_definition do |line|
+      line.hash_test :regexp => /test/, :captures => []
+    end
+    
+    @format_instance = @first_file_format.new
+    @format_instance.should have(1).line_definitions
+    @format_instance.line_definitions[:hash_test].should_not be_nil
+  end
+  
+  it "should specift lines directly" do
+    @first_file_format.new.should have(0).line_definitions    
+    
+    @first_file_format.format_definition.direct_test do |line|
+      line.regexp   = /test/
+    end 
+    
+    @first_file_format.new.line_definitions[:direct_test].should_not be_nil
+  end
+ 
+  it "specify lines with a block" do
+    
+    @first_file_format.new.should have(0).line_definitions    
+    
+    @first_file_format.format_definition do |format|
+      format.block_test do |line|
+        line.regexp   = /test/
+        line.captures = []
+      end
+    end 
+    
+    @first_file_format.new.line_definitions[:block_test].should_not be_nil
+  end
+  
+  it "should define lines only for itself" do
+    
+    @first_file_format.format_definition do |line|
+      line.first_test :regexp => /test/, :captures => []
+    end
+    
+    @second_file_format.format_definition do |line|
+      line.second_test :regexp => /test/, :captures => []
+    end
+
+    @first_file_format.new.should have(1).line_definitions    
+    @first_file_format.new.line_definitions[:first_test].should_not be_nil
+    @second_file_format.new.should have(1).line_definitions        
+    @second_file_format.new.line_definitions[:second_test].should_not be_nil
+  end  
+end
+
+describe RequestLogAnalyzer::FileFormat, :load do
+
+  include RequestLogAnalyzerSpecHelper
+
+  it "should load a predefined file format from the /file_format dir" do
+    @file_format = RequestLogAnalyzer::FileFormat.load(:rails)
+    @file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Rails)
+  end
+  
+  it "should load a provided format file" do
+    @file_format = RequestLogAnalyzer::FileFormat.load(format_file(:spec_format))
+    @file_format.should be_kind_of(SpecFormat)
+  end
+  
+end

data/spec/file_formats/spec_format.rb

@@ -0,0 +1,26 @@
+class SpecFormat < RequestLogAnalyzer::FileFormat
+
+  format_definition.first do |line|
+    line.header = true
+    line.teaser = /processing /
+    line.regexp = /processing request (\d+)/
+    line.captures = [{ :name => :request_no, :type => :integer, :anonymize => :slightly }]    
+  end
+  
+  format_definition.test do |line|
+    line.teaser = /testing /
+    line.regexp = /testing is (\w+)/
+    line.captures = [{ :name => :test_capture, :type => :string, :anonymize => true}]
+  end
+  
+  format_definition.last do |line|
+    line.footer = true
+    line.teaser = /finishing /
+    line.regexp = /finishing request (\d+)/
+    line.captures = [{ :name => :request_no, :type => :integer}]
+  end
+  
+  report do |analyze|
+    analyze.category :test_capture, :title => 'What is testing exactly?'
+  end
+end