request-log-analyzer 1.1.3 → 1.1.4

data/README.rdoc

@@ -3,7 +3,7 @@
 This is a simple command line tool to analyze request log files of both Rails and
 Merb to produce a performance report. Its purpose is to find what actions are best candidates for optimization.
 
-* Analyzes Rails log files (all versions)
+* Analyzes Rails log files (all versions), Merb logs, or any other log format
 * Can combine multiple files (handy if you are using logrotate)
 * Uses several metrics, including cumulative request time, average request time, process blockers, database and rendering time, HTTP methods and statuses, Rails action cache statistics, etc.) (Sample output: http://wiki.github.com/wvanbergen/request-log-analyzer/sample-output)
 * Low memory footprint (server-safe)

data/bin/request-log-analyzer

@@ -1,6 +1,7 @@
 #!/usr/bin/ruby
 require File.dirname(__FILE__) + '/../lib/request_log_analyzer'
 require File.dirname(__FILE__) + '/../lib/cli/command_line_arguments'
+require File.dirname(__FILE__) + '/../lib/cli/progressbar'
 require File.dirname(__FILE__) + '/../lib/cli/tools'
 
 # Parse the arguments given via commandline

data/lib/request_log_analyzer.rb

@@ -1,15 +1,25 @@
 require 'date'
-require File.dirname(__FILE__) + '/cli/progressbar'
 
+# RequestLogAnalyzer is the base namespace in which all functionality of RequestLogAnalyzer is implemented.
+#
+# - This module itselfs contains some functions to help with class and source file loading.
+# - The actual application resides in the RequestLogAnalyzer::Controller class.
 module RequestLogAnalyzer
-   
+  
+  # The current version of request-log-analyzer.
+  # This will be diplayed in output reports etc.  
   VERSION = '1.1'
   
+  # Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
+  # <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.
   def self.const_missing(const)
     load_default_class_file(RequestLogAnalyzer, const)
   end
     
-  # Function to implement 
+  # Loads constants that reside in the RequestLogAnalyzer tree using the constant name
+  # and its base constant to determine the filename.
+  # <tt>base</tt>:: The base constant to load the constant from. This should be Foo when the constant Foo::Bar is being loaded.
+  # <tt>const</tt>:: The constant to load from the base constant as a string or symbol. This should be 'Bar' or :Bar when the constant Foo::Bar is being loaded.
   def self.load_default_class_file(base, const)
     path     = to_underscore(base.to_s)
     basename = to_underscore(const.to_s)
@@ -19,14 +29,16 @@ module RequestLogAnalyzer
   end
 
   # Convert a string/symbol in camelcase (RequestLogAnalyzer::Controller) to underscores (request_log_analyzer/controller)
+  # This function can be used to load the file (using require) in which the given constant is defined.
+  # <tt>str</tt>:: The string to convert in the following format: <tt>ModuleName::ClassName</tt>
   def self.to_underscore(str)
     str.to_s.gsub(/::/, '/').gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').gsub(/([a-z\d])([A-Z])/,'\1_\2').tr("-", "_").downcase
   end  
 
-  # Convert a string/symbol in underscores (request_log_analyzer/controller) to camelcase (RequestLogAnalyzer::Controller)    
+  # Convert a string/symbol in underscores (<tt>request_log_analyzer/controller</tt>) to camelcase 
+  # (<tt>RequestLogAnalyzer::Controller</tt>). This can be used to find the class that is defined in a given filename.
+  # <tt>str</tt>:: The string to convert in the following format: <tt>module_name/class_name</tt>
   def self.to_camelcase(str)
-    str.to_s.to_s.gsub(/\/(.?)/) { "::" + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
+    str.to_s.gsub(/\/(.?)/) { "::" + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
   end
 end
-
-

data/lib/request_log_analyzer/file_format/merb.rb

@@ -39,8 +39,8 @@ module RequestLogAnalyzer::FileFormat
     
     report do |analyze|
       analyze.timespan :line_type => :started
+      analyze.frequency :category => REQUEST_CATEGORIZER, :amount => 20, :title => "Top 20 by hits"
       analyze.hourly_spread :line_type => :started
-      
       analyze.duration :dispatch_time, :category => REQUEST_CATEGORIZER, :title => 'Request dispatch duration'
       # analyze.duration :action_time, :category => REQUEST_CATEGORIZER, :title => 'Request action duration'
       # analyze.duration :after_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request after_filter duration'

data/lib/request_log_analyzer/output/fixed_width.rb

@@ -162,14 +162,18 @@ module RequestLogAnalyzer::Output
                 bar << colorize(characters[:block] * (width.to_f * (row[index].to_f - column[:treshold])).round, :red) 
                 row_values.push(bar) 
               else
+                # Create a bar by combining block characters
                 row_values.push(characters[:block] * (width.to_f * row[index].to_f).round) 
               end
             else
+              # Too few characters for a ratio bar. Display nothing
               row_values.push('')
             end
           else
-            alignment = (columns[index][:align] == :right ? '' : '-')        
-            row_values.push("%#{alignment}#{width}s" % row[index].to_s[0...width])
+            alignment = (columns[index][:align] == :right ? '' : '-')
+            cell_value = "%#{alignment}#{width}s" % row[index].to_s[0...width]
+            cell_value = colorize(cell_value, :bold, :brown) if columns[index][:highlight]
+            row_values.push(cell_value)
           end
         end
         puts row_values.join(style[:cell_separator] ? " #{characters[:vertical_line]} " : ' ')      

data/lib/request_log_analyzer/source.rb

@@ -1,12 +1,27 @@
+# The RequestLogAnalyzer::Source module contains all functionality that loads requests from a given source
+# and feed them to the pipeline for further processing. The requests (see RequestLogAnalyzer::Request) that
+# will be parsed from a source, will be piped throug filters (see RequestLogAnalyzer::Filter) and are then 
+# fed to an aggregator (see RequestLogAnalyzer::Aggregator). The source instance is thus the beginning of 
+# the RequestLogAnalyzer chain.
+#
+# - The base class for all sources is RequestLogAnalyzer::Source::Base. All source classes should inherit from this class.
+# - Currently, RequestLogAnalyzer::Source::LogParser is the only implemented source.
 module RequestLogAnalyzer::Source
 
+  # Loads constants that reside in the RequestLogAnalyzer::Source namespace. This function uses
+  # RequestLogAnalyzer::load_default_class_file to load the file in which the constant is declared.
+  # <tt>const</tt>:: The constant to load in the RequestLogAnalyzer::Source namespace.
   def self.const_missing(const)
     RequestLogAnalyzer::load_default_class_file(self, const)
   end
   
-  # Base Source class. All other sources inherit from this class
+  # The base Source class. All other sources should inherit from this class.
+  #
+  # A source implememtation should at least implement the each_request method, which should yield
+  # RequestLogAnalyzer::Request instances that will be fed through the pipleine.
   class Base
     
+    # Make the Spurce instance aware of the current file format
     include RequestLogAnalyzer::FileFormat::Awareness
 
     # A hash of options
@@ -24,23 +39,29 @@ module RequestLogAnalyzer::Source
     # The number of skipped lines because of warnings
     attr_reader :skipped_lines
 
-    # Base source class used to filter input requests.
-
-    # Initializer
-    # <tt>format</tt> The file format
-    # <tt>options</tt> Are passed to the filters.
+    # Initializer, which will register the file format and save any options given as a hash.
+    # <tt>format</tt>:: The file format instance
+    # <tt>options</tt>:: A hash of options that can be used by a specific Source implementation
     def initialize(format, options = {})
       @options    = options
       register_file_format(format)
     end
     
+    # The prepare method is called before the RequestLogAnalyzer::Source::Base#each_request method is called.
+    # Use this method to implement any initialization that should occur before this source can produce Request
+    # instances.
     def prepare
     end
     
-    def each_request(&block)
+    # This function is called to actually produce the requests that will be send into the pipeline. 
+    # The implementation should yield instances of RequestLogAnalyzer::Request.
+    # <tt>options</tt>:: A Hash of options that can be used in the implementation.
+    def each_request(options = {}, &block) # :yields: request
       return true
     end
 
+    # This function is called after RequestLogAnalyzer::Source::Base#each_request finished. Any code to
+    # wrap up, free resources, etc. can be put in this method.
     def finalize
     end
 

data/lib/request_log_analyzer/source/log_parser.rb

@@ -7,18 +7,24 @@ module RequestLogAnalyzer::Source
   # De order in which lines occur is used to combine lines to a single request. If these lines 
   # are mixed, requests cannot be combined properly. This can be the case if data is written to 
   # the log file simultaneously by different mongrel processes. This problem is detected by the 
-  # parser, but the requests that are mixed up cannot be parsed. It will emit warnings when this 
-  # occurs.
+  # parser. It will emit warnings when this occurs. LogParser supports multiple parse strategies 
+  # that deal differently with this problem.
   class LogParser < Base
 
+    # The default parse strategy that will be used to parse the input.
     DEFAULT_PARSE_STRATEGY = 'assume-correct'
+    
+    # All available parse strategies.
     PARSE_STRATEGIES = ['cautious', 'assume-correct']
 
     attr_reader :source_files
 
-    # Initializes the parser instance.
+    # Initializes the log file parser instance.
     # It will apply the language specific FileFormat module to this instance. It will use the line
-    # definitions in this module to parse any input.
+    # definitions in this module to parse any input that it is given (see parse_io).
+    #
+    # <tt>format</tt>:: The current file format instance
+    # <tt>options</tt>:: A hash of options that are used by the parser
     def initialize(format, options = {})      
       @line_definitions = {}
       @options          = options
@@ -35,7 +41,11 @@ module RequestLogAnalyzer::Source
       self.register_file_format(format)
     end
     
-    def each_request(options = {}, &block)
+    # Reads the input, which can either be a file, sequence of files or STDIN to parse
+    # lines specified in the FileFormat. This lines will be combined into Request instances,
+    # that will be yielded. The actual parsing occurs in the parse_io method.
+    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
+    def each_request(options = {}, &block) # :yields: request
       
       case @source_files
       when IO;     
@@ -50,28 +60,46 @@ module RequestLogAnalyzer::Source
       end
     end
 
-    # Parses a list of consequent files of the same format
-    def parse_files(files, options = {}, &block)
+    # Parses a list of subsequent files of the same format, by calling parse_file for every
+    # file in the array.
+    # <tt>files</tt>:: The Array of files that should be parsed
+    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
+    def parse_files(files, options = {}, &block) # :yields: request
       files.each { |file| parse_file(file, options, &block) }
     end
 
-    # Parses a file. 
-    # Creates an IO stream for the provided file, and sends it to parse_io for further handling
+    # Parses a log file. Creates an IO stream for the provided file, and sends it to parse_io for
+    # further handling. This method supports progress updates that can be used to display a progressbar
+    # <tt>file</tt>:: The file that should be parsed.
+    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.    
     def parse_file(file, options = {}, &block)
       @progress_handler.call(:started, file) if @progress_handler
       File.open(file, 'r') { |f| parse_io(f, options, &block) }
       @progress_handler.call(:finished, file) if @progress_handler
     end
 
+
+    # Parses an IO stream. It will simply call parse_io. This function does not support progress updates
+    # because the length of a stream is not known.
+    # <tt>stream</tt>:: The IO stream that should be parsed.
+    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.    
     def parse_stream(stream, options = {}, &block)
       parse_io(stream, options, &block)
     end
 
-    # Finds a log line and then parses the information in the line.
-    # Yields a hash containing the information found. 
-    # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
-    # Yeilds a Hash when it encounters a chunk of information.
-    def parse_io(io, options = {}, &block)
+    # This method loops over each line of the input stream. It will try to parse this line as any of
+    # the lines that are defined by the current file format (see RequestLogAnalyazer::FileFormat).
+    # It will then combine these parsed line into requests using heuristics. These requests (see
+    # RequestLogAnalyzer::Request) will then be yielded for further processing in the pipeline.
+    #
+    # - RequestLogAnalyzer::LineDefinition#matches is called to test if a line matches a line definition of the file format.
+    # - update_current_request is used to combine parsed lines into requests using heuristics.
+    # - The method will yield progress updates if a progress handler is installed using progress=
+    # - The method will yield parse warnings if a warning handler is installed using warning=
+    #
+    # <tt>io</tt>:: The IO instance to use as source
+    # <tt>options</tt>:: A hash of options that can be used by the parser.
+    def parse_io(io, options = {}, &block) # :yields: request
 
       @current_io = io
       @current_io.each_line do |line|
@@ -95,19 +123,27 @@ module RequestLogAnalyzer::Source
       @current_io = nil
     end
 
-    # Add a block to this method to install a progress handler while parsing
+    # Add a block to this method to install a progress handler while parsing.
+    # <tt>proc</tt>:: The proc that will be called to handle progress update messages
     def progress=(proc)
       @progress_handler = proc
     end
 
-    # Add a block to this method to install a warning handler while parsing
+    # Add a block to this method to install a warning handler while parsing,
+    # <tt>proc</tt>:: The proc that will be called to handle parse warning messages    
     def warning=(proc)
       @warning_handler = proc
     end
 
-    # This method is called by the parser if it encounteres any problems.
-    # It will call the warning handler. The default controller will pass all warnings to every
-    # aggregator that is registered and running
+    # This method is called by the parser if it encounteres any parsing problems.
+    # It will call the installed warning handler if any. 
+    #
+    # By default, RequestLogAnalyzer::Controller will install a warning handler 
+    # that will pass the warnings to each aggregator so they can do something useful
+    # with it.
+    #
+    # <tt>type</tt>:: The warning type (a Symbol)
+    # <tt>message</tt>:: A message explaining the warning
     def warn(type, message)
       @warning_handler.call(type, message, @current_io.lineno) if @warning_handler
     end
@@ -118,13 +154,25 @@ module RequestLogAnalyzer::Source
     # new request when a header line is encountered en will emit the request when a footer line 
     # is encountered.
     #
+    # Combining the lines is done using heuristics. Problems can occur in this process. The
+    # current parse strategy defines how these cases are handled.
+    #
+    # When using the 'assume-correct' parse strategy (default):
+    # - Every line that is parsed before a header line is ignored as it cannot be included in 
+    #   any request. It will emit a :no_current_request warning.
+    # - If a header line is found before the previous requests was closed, the previous request 
+    #   will be yielded and a new request will be started.
+    #
+    # When using the 'cautious' parse strategy:
     # - Every line that is parsed before a header line is ignored as it cannot be included in 
     #   any request. It will emit a :no_current_request warning.
     # - A header line that is parsed before a request is closed by a footer line, is a sign of
-    #   an unprpertly ordered file. All data that is gathered for the request until then is 
-    #   discarded, the next request is ignored as well and a :unclosed_request warning is
+    #   an unproperly ordered file. All data that is gathered for the request until then is 
+    #   discarded and the next request is ignored as well. An :unclosed_request warning is
     #   emitted.
-    def update_current_request(request_data, &block)
+    #
+    # <tt>request_data</tt>:: A hash of data that was parsed from the last line.
+    def update_current_request(request_data, &block) # :yields: request
       if header_line?(request_data)
         unless @current_request.nil?
           case options[:parse_strategy]
@@ -153,21 +201,28 @@ module RequestLogAnalyzer::Source
       end
     end
     
-    # Handles the parsed request by calling the request handler.
-    # The default controller will send the request to every running aggegator.
-    def handle_request(request, &block)
+    # Handles the parsed request by sending it into the pipeline. 
+    #
+    # - It will call RequestLogAnalyzer::Request#validate on the request instance
+    # - It will send the request into the pipeline, checking whether it was accepted by all the filters.
+    # - It will update the parsed_requests and skipped_requests variables accordingly
+    #
+    # <tt>request</tt>:: The parsed request instance (RequestLogAnalyzer::Request)
+    def handle_request(request, &block) # :yields: request
       @parsed_requests += 1
       request.validate
       accepted = block_given? ? yield(request) : true
       @skipped_requests += 1 if not accepted
     end    
 
-    # Checks whether a given line hash is a header line.
+    # Checks whether a given line hash is a header line according to the current file format.
+    # <tt>hash</tt>:: A hash of data that was parsed from the line.    
     def header_line?(hash)
       hash[:line_definition].header
     end
 
-    # Checks whether a given line hash is a footer line.    
+    # Checks whether a given line hash is a footer line  according to the current file format.    
+    # <tt>hash</tt>:: A hash of data that was parsed from the line.    
     def footer_line?(hash)
       hash[:line_definition].footer
     end 

data/lib/request_log_analyzer/tracker/duration.rb

@@ -49,9 +49,11 @@ module RequestLogAnalyzer::Tracker
         duration = options[:duration].respond_to?(:call) ? options[:duration].call(request) : request[options[:duration]]
   
         if !duration.nil? && !category.nil?
-          @categories[category] ||= {:hits => 0, :cumulative => 0.0}
+          @categories[category] ||= {:hits => 0, :cumulative => 0.0, :min => duration, :max => duration }
           @categories[category][:hits] += 1
           @categories[category][:cumulative] += duration
+          @categories[category][:min] = duration if duration < @categories[category][:min]
+          @categories[category][:max] = duration if duration > @categories[category][:max]   
         end
       end
     end
@@ -63,7 +65,15 @@ module RequestLogAnalyzer::Tracker
     def cumulative_duration(cat)
       categories[cat][:cumulative]
     end
-    
+
+    def min_duration(cat)
+      categories[cat][:min]
+    end
+
+    def max_duration(cat)
+      categories[cat][:max]
+    end
+
     def average_duration(cat)
       categories[cat][:cumulative] / categories[cat][:hits]  
     end
@@ -106,11 +116,16 @@ module RequestLogAnalyzer::Tracker
       output.title(options[:title])
     
       top_categories = @categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }.slice(0...amount)
-      output.table({:title => 'Category', :width => :rest}, {:title => 'Hits', :align => :right, :min_width => 4}, 
-            {:title => 'Cumulative', :align => :right, :min_width => 10}, {:title => 'Average', :align => :right, :min_width => 8}) do |rows|
+      output.table({:title => 'Category', :width => :rest}, 
+            {:title => 'Hits',       :align => :right, :highlight => (options[:sort] == :hits), :min_width => 4}, 
+            {:title => 'Cumulative', :align => :right, :highlight => (options[:sort] == :cumulative), :min_width => 10}, 
+            {:title => 'Average',    :align => :right, :highlight => (options[:sort] == :average), :min_width => 8},
+            {:title => 'Min',        :align => :right, :highlight => (options[:sort] == :min)}, 
+            {:title => 'Max',        :align => :right, :highlight => (options[:sort] == :max)}) do |rows|
       
         top_categories.each do |(cat, info)|
-          rows << [cat, info[:hits], "%0.02fs" % info[:cumulative], "%0.02fs" % (info[:cumulative] / info[:hits])]
+          rows << [cat, info[:hits], "%0.02fs" % info[:cumulative], "%0.02fs" % (info[:cumulative] / info[:hits]),
+                    "%0.02fs" % info[:min], "%0.02fs" % info[:max]]
         end        
       end
 
@@ -125,11 +140,11 @@ module RequestLogAnalyzer::Tracker
       options[:report].each do |report|
         case report
         when :average
-          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by average time") { |cat| cat[:cumulative] / cat[:hits] }  
+          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by average time", :sort => :average) { |cat| cat[:cumulative] / cat[:hits] }  
         when :cumulative
-          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by cumulative time") { |cat| cat[:cumulative] }
+          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by cumulative time", :sort => :cumulative) { |cat| cat[:cumulative] }
         when :hits
-          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by hits") { |cat| cat[:hits] }
+          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by hits", :sort => :hits) { |cat| cat[:hits] }
         else
           raise "Unknown duration report specified: #{report}!"
         end

data/spec/unit/tracker/duration_tracker_spec.rb

@@ -41,6 +41,15 @@ describe RequestLogAnalyzer::Tracker::Duration, 'static category' do
     @tracker.average_duration('b').should == 0.35
   end
   
+  it "should set min and max duration correctly" do
+    @tracker.update(request(:category => 'a', :duration => 0.2))
+    @tracker.update(request(:category => 'b', :duration => 0.3))
+    @tracker.update(request(:category => 'b', :duration => 0.4))      
+    
+    @tracker.min_duration('b').should == 0.3
+    @tracker.max_duration('b').should == 0.4
+  end  
+  
 end
 
 describe RequestLogAnalyzer::Tracker::Duration, 'dynamic category' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: request-log-analyzer
 version: !ruby/object:Gem::Version 
-  version: 1.1.3
+  version: 1.1.4
 platform: ruby
 authors: 
 - Willem van Bergen
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-01-29 00:00:00 +01:00
+date: 2009-02-08 00:00:00 +01:00
 default_executable: request-log-analyzer
 dependencies: []
 
@@ -143,7 +143,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: r-l-a
-rubygems_version: 1.2.0
+rubygems_version: 1.3.1
 signing_key: 
 specification_version: 2
 summary: A command line tool to analyze Rails logs