request-log-analyzer 1.3.7 → 1.4.0

data/lib/request_log_analyzer/source.rb

@@ -1,7 +1,7 @@
 # The RequestLogAnalyzer::Source module contains all functionality that loads requests from a given source
 # and feed them to the pipeline for further processing. The requests (see RequestLogAnalyzer::Request) that
-# will be parsed from a source, will be piped throug filters (see RequestLogAnalyzer::Filter) and are then 
-# fed to an aggregator (see RequestLogAnalyzer::Aggregator). The source instance is thus the beginning of 
+# will be parsed from a source, will be piped throug filters (see RequestLogAnalyzer::Filter) and are then
+# fed to an aggregator (see RequestLogAnalyzer::Aggregator). The source instance is thus the beginning of
 # the RequestLogAnalyzer chain.
 #
 # - The base class for all sources is RequestLogAnalyzer::Source::Base. All source classes should inherit from this class.
@@ -14,13 +14,13 @@ module RequestLogAnalyzer::Source
   def self.const_missing(const)
     RequestLogAnalyzer::load_default_class_file(self, const)
   end
-  
+
   # The base Source class. All other sources should inherit from this class.
   #
   # A source implememtation should at least implement the each_request method, which should yield
   # RequestLogAnalyzer::Request instances that will be fed through the pipleine.
   class Base
-    
+
     # A hash of options
     attr_reader :options
 
@@ -49,14 +49,14 @@ module RequestLogAnalyzer::Source
       @options     = options
       @file_format = format
     end
-    
+
     # The prepare method is called before the RequestLogAnalyzer::Source::Base#each_request method is called.
     # Use this method to implement any initialization that should occur before this source can produce Request
     # instances.
     def prepare
     end
-    
-    # This function is called to actually produce the requests that will be send into the pipeline. 
+
+    # This function is called to actually produce the requests that will be send into the pipeline.
     # The implementation should yield instances of RequestLogAnalyzer::Request.
     # <tt>options</tt>:: A Hash of options that can be used in the implementation.
     def each_request(options = {}, &block) # :yields: request

data/lib/request_log_analyzer/source/database_loader.rb

@@ -2,7 +2,7 @@ require 'rubygems'
 require 'activerecord'
 
 module RequestLogAnalyzer::Source
-  
+
   # Active Resource hook
   class Request < ActiveRecord::Base
     has_many :completed_lines
@@ -40,7 +40,7 @@ module RequestLogAnalyzer::Source
       @parsed_requests  = 0
       @requests         = []
     end
-    
+
     # Reads the input, which can either be a file, sequence of files or STDIN to parse
     # lines specified in the FileFormat. This lines will be combined into Request instances,
     # that will be yielded. The actual parsing occurs in the parse_io method.
@@ -52,10 +52,10 @@ module RequestLogAnalyzer::Source
         RequestLogAnalyzer::Source::Request.find(:all).each do |request|
           @parsed_requests += 1
           @progress_handler.call(:progress, @parsed_requests) if @progress_handler
-          
+
           yield request.convert(self.file_format)
         end
-      
+
       @progress_handler.call(:finished, @source_files) if @progress_handler
     end
 
@@ -66,15 +66,15 @@ module RequestLogAnalyzer::Source
     end
 
     # Add a block to this method to install a warning handler while parsing,
-    # <tt>proc</tt>:: The proc that will be called to handle parse warning messages    
+    # <tt>proc</tt>:: The proc that will be called to handle parse warning messages
     def warning=(proc)
       @warning_handler = proc
     end
 
     # This method is called by the parser if it encounteres any parsing problems.
-    # It will call the installed warning handler if any. 
+    # It will call the installed warning handler if any.
     #
-    # By default, RequestLogAnalyzer::Controller will install a warning handler 
+    # By default, RequestLogAnalyzer::Controller will install a warning handler
     # that will pass the warnings to each aggregator so they can do something useful
     # with it.
     #

data/lib/request_log_analyzer/source/log_parser.rb

@@ -1,13 +1,13 @@
 module RequestLogAnalyzer::Source
-  
+
   # The LogParser class reads log data from a given source and uses a file format definition
-  # to parse all relevent information about requests from the file. A FileFormat module should 
+  # to parse all relevent information about requests from the file. A FileFormat module should
   # be provided that contains the definitions of the lines that occur in the log data.
   #
-  # De order in which lines occur is used to combine lines to a single request. If these lines 
-  # are mixed, requests cannot be combined properly. This can be the case if data is written to 
-  # the log file simultaneously by different mongrel processes. This problem is detected by the 
-  # parser. It will emit warnings when this occurs. LogParser supports multiple parse strategies 
+  # De order in which lines occur is used to combine lines to a single request. If these lines
+  # are mixed, requests cannot be combined properly. This can be the case if data is written to
+  # the log file simultaneously by different mongrel processes. This problem is detected by the
+  # parser. It will emit warnings when this occurs. LogParser supports multiple parse strategies
   # that deal differently with this problem.
   class LogParser < Base
 
@@ -15,7 +15,7 @@ module RequestLogAnalyzer::Source
 
     # The default parse strategy that will be used to parse the input.
     DEFAULT_PARSE_STRATEGY = 'assume-correct'
-    
+
     # All available parse strategies.
     PARSE_STRATEGIES = ['cautious', 'assume-correct']
 
@@ -33,33 +33,38 @@ module RequestLogAnalyzer::Source
       @parsed_requests  = 0
       @skipped_lines    = 0
       @skipped_requests = 0
+      @current_request  = nil
+      @current_source   = nil
       @current_file     = nil
       @current_lineno   = nil
       @source_files     = options[:source_files]
-      
+      @progress_handler = nil
+
       @options[:parse_strategy] ||= DEFAULT_PARSE_STRATEGY
       raise "Unknown parse strategy" unless PARSE_STRATEGIES.include?(@options[:parse_strategy])
     end
-    
+
     # Reads the input, which can either be a file, sequence of files or STDIN to parse
     # lines specified in the FileFormat. This lines will be combined into Request instances,
     # that will be yielded. The actual parsing occurs in the parse_io method.
     # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
     def each_request(options = {}, &block) # :yields: :request, request
-      
+
       case @source_files
       when IO
-        puts "Parsing from the standard input. Press CTRL+C to finish." # FIXME: not here
-        parse_stream(@source_files, options, &block) 
+        if @source_files == $stdin
+          puts "Parsing from the standard input. Press CTRL+C to finish." # FIXME: not here
+        end
+        parse_stream(@source_files, options, &block)
       when String
-        parse_file(@source_files, options, &block) 
+        parse_file(@source_files, options, &block)
       when Array
-        parse_files(@source_files, options, &block) 
+        parse_files(@source_files, options, &block)
       else
         raise "Unknown source provided"
       end
     end
-    
+
     # Make sure the Enumerable methods work as expected
     alias_method :each, :each_request
 
@@ -70,12 +75,12 @@ module RequestLogAnalyzer::Source
     def parse_files(files, options = {}, &block) # :yields: request
       files.each { |file| parse_file(file, options, &block) }
     end
-    
+
     # Check if a file has a compressed extention in the filename.
     # If recognized, return the command string used to decompress the file
     def decompress_file?(filename)
       nice_command = "nice -n 5"
-      
+
       return "#{nice_command} gunzip -c -d #{filename}" if filename.match(/\.tar.gz$/) || filename.match(/\.tgz$/) || filename.match(/\.gz$/)
       return "#{nice_command} bunzip2 -c -d #{filename}" if filename.match(/\.bz2$/)
       return "#{nice_command} unzip -p #{filename}" if filename.match(/\.zip$/)
@@ -96,22 +101,22 @@ module RequestLogAnalyzer::Source
 
       @current_source = File.expand_path(file)
       @source_changes_handler.call(:started, @current_source) if @source_changes_handler
-      
+
       if decompress_file?(file).empty?
 
         @progress_handler = @dormant_progress_handler
         @progress_handler.call(:started, file) if @progress_handler
-        
+
         File.open(file, 'r') { |f| parse_io(f, options, &block) }
-        
+
         @progress_handler.call(:finished, file) if @progress_handler
         @progress_handler = nil
       else
         IO.popen(decompress_file?(file), 'r') { |f| parse_io(f, options, &block) }
       end
-      
+
       @source_changes_handler.call(:finished, @current_source) if @source_changes_handler
-      
+
       @current_source = nil
 
     end
@@ -119,7 +124,7 @@ module RequestLogAnalyzer::Source
     # Parses an IO stream. It will simply call parse_io. This function does not support progress updates
     # because the length of a stream is not known.
     # <tt>stream</tt>:: The IO stream that should be parsed.
-    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.    
+    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
     def parse_stream(stream, options = {}, &block)
       parse_io(stream, options, &block)
     end
@@ -140,15 +145,15 @@ module RequestLogAnalyzer::Source
       @current_lineno = 1
       while line = io.gets
         @progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno % 255 == 0
-        
+
         if request_data = file_format.parse_line(line) { |wt, message| warn(wt, message) }
           @parsed_lines += 1
           update_current_request(request_data.merge(:source => @current_source, :lineno => @current_lineno), &block)
         end
-        
+
         @current_lineno += 1
       end
-      
+
       warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
       @current_lineno = nil
     end
@@ -164,7 +169,7 @@ module RequestLogAnalyzer::Source
     def warning=(proc)
       @warning_handler = proc
     end
-    
+
     # Add a block to this method to install a source change handler while parsing,
     # <tt>proc</tt>:: The proc that will be called to handle source changes
     def source_changes=(proc)
@@ -172,9 +177,9 @@ module RequestLogAnalyzer::Source
     end
 
     # This method is called by the parser if it encounteres any parsing problems.
-    # It will call the installed warning handler if any. 
+    # It will call the installed warning handler if any.
     #
-    # By default, RequestLogAnalyzer::Controller will install a warning handler 
+    # By default, RequestLogAnalyzer::Controller will install a warning handler
     # that will pass the warnings to each aggregator so they can do something useful
     # with it.
     #
@@ -186,24 +191,24 @@ module RequestLogAnalyzer::Source
 
     protected
 
-    # Combines the different lines of a request into a single Request object. It will start a 
-    # new request when a header line is encountered en will emit the request when a footer line 
+    # Combines the different lines of a request into a single Request object. It will start a
+    # new request when a header line is encountered en will emit the request when a footer line
     # is encountered.
     #
     # Combining the lines is done using heuristics. Problems can occur in this process. The
     # current parse strategy defines how these cases are handled.
     #
     # When using the 'assume-correct' parse strategy (default):
-    # - Every line that is parsed before a header line is ignored as it cannot be included in 
+    # - Every line that is parsed before a header line is ignored as it cannot be included in
     #   any request. It will emit a :no_current_request warning.
-    # - If a header line is found before the previous requests was closed, the previous request 
+    # - If a header line is found before the previous requests was closed, the previous request
     #   will be yielded and a new request will be started.
     #
     # When using the 'cautious' parse strategy:
-    # - Every line that is parsed before a header line is ignored as it cannot be included in 
+    # - Every line that is parsed before a header line is ignored as it cannot be included in
     #   any request. It will emit a :no_current_request warning.
     # - A header line that is parsed before a request is closed by a footer line, is a sign of
-    #   an unproperly ordered file. All data that is gathered for the request until then is 
+    #   an unproperly ordered file. All data that is gathered for the request until then is
     #   discarded and the next request is ignored as well. An :unclosed_request warning is
     #   emitted.
     #
@@ -230,7 +235,7 @@ module RequestLogAnalyzer::Source
           @current_request << request_data
           if footer_line?(request_data)
             handle_request(@current_request, &block) # yield @current_request
-            @current_request = nil 
+            @current_request = nil
           end
         else
           @skipped_lines += 1
@@ -238,8 +243,8 @@ module RequestLogAnalyzer::Source
         end
       end
     end
-    
-    # Handles the parsed request by sending it into the pipeline. 
+
+    # Handles the parsed request by sending it into the pipeline.
     #
     # - It will call RequestLogAnalyzer::Request#validate on the request instance
     # - It will send the request into the pipeline, checking whether it was accepted by all the filters.
@@ -251,19 +256,19 @@ module RequestLogAnalyzer::Source
       request.validate
       accepted = block_given? ? yield(request) : true
       @skipped_requests += 1 unless accepted
-    end    
+    end
 
     # Checks whether a given line hash is a header line according to the current file format.
-    # <tt>hash</tt>:: A hash of data that was parsed from the line.    
+    # <tt>hash</tt>:: A hash of data that was parsed from the line.
     def header_line?(hash)
       hash[:line_definition].header
     end
 
-    # Checks whether a given line hash is a footer line  according to the current file format.    
-    # <tt>hash</tt>:: A hash of data that was parsed from the line.    
+    # Checks whether a given line hash is a footer line  according to the current file format.
+    # <tt>hash</tt>:: A hash of data that was parsed from the line.
     def footer_line?(hash)
       hash[:line_definition].footer
-    end 
+    end
   end
 
 end

data/lib/request_log_analyzer/tracker.rb

@@ -16,13 +16,13 @@ module RequestLogAnalyzer::Tracker
   #
   # For example :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
   class Base
-    
+
     attr_reader :options
-    
+
     # Initialize the class
     # Note that the options are only applicable if should_update? is not overwritten
     # by the inheriting class.
-    # 
+    #
     # === Options
     # * <tt>:if</tt> Handle request if this proc is true for the handled request.
     # * <tt>:unless</tt> Handle request if this proc is false for the handled request.
@@ -31,7 +31,7 @@ module RequestLogAnalyzer::Tracker
       @options = options
       setup_should_update_checks!
     end
-    
+
     # Sets up the tracker's should_update? checks.
     def setup_should_update_checks!
       @should_update_checks = []
@@ -42,24 +42,34 @@ module RequestLogAnalyzer::Tracker
       @should_update_checks.push( lambda { |request| !request[options[:unless]] }) if options[:unless].kind_of?(Symbol)
     end
     
+    # Creates a lambda expression to return a static field from a request. If the 
+    # argument already is a lambda exprssion, it will simply return the argument.
+    def create_lambda(arg)
+      case arg
+      when Proc   then arg
+      when Symbol then lambda { |request| request[arg] }
+      else raise "Canot create a lambda expression from this argument: #{arg.inspect}!"
+      end
+    end
+
     # Hook things that need to be done before running here.
     def prepare
     end
-    
+
     # Will be called with each request.
     # <tt>request</tt> The request to track data in.
     def update(request)
     end
-    
+
     # Hook things that need to be done after running here.
     def finalize
     end
-    
+
     # Determine if we should run the update function at all.
     # Usually the update function will be heavy, so a light check is done here
     # determining if we need to call update at all.
     #
-    # Default this checks if defined: 
+    # Default this checks if defined:
     #  * :line_type is also in the request hash.
     #  * :if is true for this request.
     #  * :unless if false for this request
@@ -68,25 +78,129 @@ module RequestLogAnalyzer::Tracker
     def should_update?(request)
       @should_update_checks.all? { |c| c.call(request) }
     end
-    
+
     # Hook report generation here.
     # Defaults to self.inspect
     # <tt>output</tt> The output object the report will be passed to.
     def report(output)
       output << self.inspect
-      output << "\n"  
+      output << "\n"
     end
-    
+
     # The title of this tracker. Used for reporting.
     def title
       self.class.to_s
     end
-    
+
     # This method is called by RequestLogAnalyzer::Aggregator:Summarizer to retrieve an
     # object with all the results of this tracker, that can be dumped to YAML format.
     def to_yaml_object
       nil
     end
-         
-  end 
+  end
+  
+  module StatisticsTracking
+    
+    # Update sthe running calculation of statistics with the newly found numeric value.
+    # <tt>category</tt>:: The category for which to update the running statistics calculations
+    # <tt>number</tt>:: The numeric value to update the calculations with.
+    def update_statistics(category, number)
+      @categories[category] ||= {:hits => 0, :sum => 0, :mean => 0.0, :sum_of_squares => 0.0, :min => number, :max => number }
+      delta = number - @categories[category][:mean]
+
+      @categories[category][:hits]           += 1
+      @categories[category][:mean]           += (delta / @categories[category][:hits])
+      @categories[category][:sum_of_squares] += delta * (number - @categories[category][:mean])
+      @categories[category][:sum]            += number
+      @categories[category][:min]             = number if number < @categories[category][:min]
+      @categories[category][:max]             = number if number > @categories[category][:max]
+    end
+    
+    # Get the number of hits of a specific category.
+    # <tt>cat</tt> The category
+    def hits(cat)
+      @categories[cat][:hits]
+    end
+
+    # Get the total duration of a specific category.
+    # <tt>cat</tt> The category
+    def sum(cat)
+      @categories[cat][:sum]
+    end
+
+    # Get the minimal duration of a specific category.
+    # <tt>cat</tt> The category
+    def min(cat)
+      @categories[cat][:min]
+    end
+
+    # Get the maximum duration of a specific category.
+    # <tt>cat</tt> The category
+    def max(cat)
+      @categories[cat][:max]
+    end
+
+    # Get the average duration of a specific category.
+    # <tt>cat</tt> The category
+    def mean(cat)
+      @categories[cat][:mean]
+    end
+
+    # Get the standard deviation of the duration of a specific category.
+    # <tt>cat</tt> The category
+    def stddev(cat)
+      Math.sqrt(variance(cat))
+    end
+
+    # Get the variance of the duration of a specific category.
+    # <tt>cat</tt> The category
+    def variance(cat)
+      return 0.0 if @categories[cat][:hits] <= 1
+      (@categories[cat][:sum_of_squares] / (@categories[cat][:hits] - 1))
+    end
+
+    # Get the average duration of a all categories.
+    def mean_overall
+      sum_overall / hits_overall
+    end
+
+    # Get the cumlative duration of a all categories.
+    def sum_overall
+      @categories.inject(0.0) { |sum, (name, cat)| sum + cat[:sum] }
+    end
+
+    # Get the total hits of a all categories.
+    def hits_overall
+      @categories.inject(0) { |sum, (name, cat)| sum + cat[:hits] }
+    end
+    
+    # Return categories sorted by a given key.
+    # <tt>by</tt> The key to sort on. This parameter can be omitted if a sorting block is provided instead
+    def sorted_by(by = nil)
+      if block_given?
+        categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }
+      else
+        categories.sort { |a, b| send(by, b[0]) <=> send(by, a[0]) }
+      end
+    end
+    
+    # Returns the column header for a statistics table to report on the statistics result
+    def statistics_header(options)
+      [
+        {:title => options[:title], :width => :rest},
+        {:title => 'Hits',   :align => :right, :highlight => (options[:highlight] == :hits),   :min_width => 4},
+        {:title => 'Sum',    :align => :right, :highlight => (options[:highlight] == :sum),    :min_width => 6},
+        {:title => 'Mean',   :align => :right, :highlight => (options[:highlight] == :mean),   :min_width => 6},
+        {:title => 'StdDev', :align => :right, :highlight => (options[:highlight] == :stddev), :min_width => 6},
+        {:title => 'Min',    :align => :right, :highlight => (options[:highlight] == :min),    :min_width => 6},
+        {:title => 'Max',    :align => :right, :highlight => (options[:highlight] == :max),    :min_width => 6}
+      ]
+    end
+    
+    # Returns a row of statistics information for a report table, given a category
+    def statistics_row(cat)
+      [cat, hits(cat), display_value(sum(cat)), display_value(mean(cat)), display_value(stddev(cat)),
+                display_value(min(cat)), display_value(max(cat))]
+    end
+  end
 end