request-log-analyzer 1.4.0 → 1.5.0

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2008 Willem van Bergen / Bart ten Brinke
+Copyright (c) 2008-2009 Willem van Bergen & Bart ten Brinke
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.rdoc

@@ -1,42 +1,37 @@
 = Request-log-analyzer
 
-This is a simple command line tool to analyze request log files of both Rails and
-Merb to produce a performance report. Its purpose is to find what actions are best candidates for optimization.
+This is a simple command line tool to analyze request log files in various formats to produce a performance report. Its purpose is to find what actions are best candidates for optimization.
 
-* Analyzes Rails log files (all versions), Merb logs, or any other log format you specify
-* Combines multiple files and decompresses compressed files (handy if you are using logrotate)
-* Uses several metrics, including cumulative request time, average request time, process blockers, database and rendering time, HTTP methods and statuses, Rails action cache statistics, etc.) (Sample output: http://wiki.github.com/wvanbergen/request-log-analyzer/sample-output)
-* Low memory footprint and reasonably fast (server-safe)
+* Analyzes Rails request logs, Merb request logs, Apache access logs and more, or parses any other log format you specify.
+* Combines multiple files and decompresses compressed files, which comes in handy if you are using logrotate.
+* Uses several metrics, including cumulative request time, mean request time, process blockers, database and rendering time, HTTP methods and statuses, Rails action cache statistics, etc.) (Sample output: http://wiki.github.com/wvanbergen/request-log-analyzer/sample-output)
+* Low memory footprint and reasonably fast, so it is safe to run on a production server.
 * MIT licensed
 
-Request log analyzer was designed and built by Willem van Bergen and Bart ten Brinke.
+See the project wiki at http://wiki.github.com/wvanbergen/request-log-analyzer for documentation and additional information.
 
-== Installation
+== Installation & basic usage
 
-Install request-log-analyzer as a Ruby gem:
+Install request-log-analyzer as a Ruby gem (you might need to run this command
+as root by prepending +sudo+ to it):
 
-  $ sudo gem install request-log-analyzer
+  $ gem install request-log-analyzer
 
-Alternatively, use the gem from the GitHub gem server:
-
-  $ sudo gem install wvanbergen-request-log-analyzer --source http://gems.github.com
-
-To get the best results out of request-log-analyzer, make sure to
-set up logging correctly: http://wiki.github.com/wvanbergen/request-log-analyzer/configure-logging
-for your application.
-
-== Usage
-
-To analyze a log file and produce a performance report, run request-log-analyzer like this:
+To analyze a Rails log file and produce a performance report, run
+request-log-analyzer like this:
 
   $ request-log-analyzer log/production.log
 
-For more details and available command line options, see the project's wiki: http://wiki.github.com/wvanbergen/request-log-analyzer/basic-usage
+For more details, other file formats, and available command line options, see the project's wiki at http://wiki.github.com/wvanbergen/request-log-analyzer
 
 == Additional information
 
-Do you have a rails application that is not performing as it should?
-If you need an expert to analyze your application, feel free to contact either Willem van Bergen (willem@railsdoctors.com) or Bart ten Brinke (bart@railsdoctors.com).
+Request-log-analyzer was designed and built by Willem van Bergen and Bart ten
+Brinke.
+
+Do you have a rails application that is not performing as it should? If you need
+an expert to analyze your application, feel free to contact either Willem van
+Bergen (willem@railsdoctors.com) or Bart ten Brinke (bart@railsdoctors.com).
 
 * Project wiki at GitHub: http://wiki.github.com/wvanbergen/request-log-analyzer
 * railsdoctors homepage: http://railsdoctors.com

data/bin/request-log-analyzer

@@ -27,14 +27,16 @@ begin
       strip.switch(:keep_junk_lines, :j)
     end
 
-    command_line.option(:format, :alias => :f, :default => 'rails')
+    command_line.option(:format, :alias => :f)
     command_line.option(:apache_format)
     command_line.option(:rails_format)
 
     command_line.option(:file, :alias => :e)
     command_line.option(:mail, :alias => :m)
+    command_line.option(:mailhost, :default => 'localhost')
     command_line.option(:parse_strategy, :default => 'assume-correct')
-    command_line.option(:dump)
+    command_line.option(:yaml)
+    command_line.option(:dump) # To be deprecated
 
     command_line.option(:aggregator, :alias => :a, :multiple => true)
 
@@ -48,7 +50,7 @@ begin
     command_line.option(:before)
 
     command_line.switch(:boring, :b)
-    command_line.option(:output, :alias => :o, :default => 'FixedWidth')
+    command_line.option(:output, :alias => :o, :default => 'fixedwidth')
     command_line.option(:report_width,  :default => terminal_width - 1)
     command_line.option(:report_amount, :default => 20)
     command_line.option(:report_sort,   :default => 'sum,mean')
@@ -68,7 +70,7 @@ rescue CommandLine::Error => e
   puts "Usage: request-log-analyzer [LOGFILES*] <OPTIONS>"
   puts
   puts "Input options:"
-  puts "  --format <format>, -f:     Uses the specified log file format. Defaults to rails."
+  puts "  --format <format>, -f:     Log file format. amazon_s3, apache, merb, mysql, rack or rails (default)."
   puts "  --after <date>             Only consider requests from <date> or later."
   puts "  --before <date>            Only consider requests before <date>."
   puts "  --select <field> <value>   Only consider requests where <field> matches <value>."
@@ -78,10 +80,11 @@ rescue CommandLine::Error => e
   puts "  --boring, -b               Output reports without ASCII colors."
   puts "  --database <filename>, -d: Creates an SQLite3 database of all the parsed request information."
   puts "  --debug                    Print debug information while parsing."
-  puts "  --file <filename>          Output to file."
+  puts "  --file <filename>          Redirect output to file."
   puts "  --mail <emailaddress>      Send report to an email address."
-  puts "  --output <format>          Output format. Supports 'HTML' and 'FixedWidth' (default)"
-  puts "  --dump <filename>          Dump the YAML formatted results in the given file"
+  puts "  --mailhost <server>        Use the given server as the SMTP server for sending email."
+  puts "  --output <format>          Output format. Supports 'html' and 'fixed_width' (default)."
+  puts "  --yaml <filename>          Dump the results in YAML format in the given file."
   puts
   puts "Examples:"
   puts "  request-log-analyzer development.log"

data/lib/request_log_analyzer/aggregator/summarizer.rb

@@ -49,6 +49,17 @@ module RequestLogAnalyzer::Aggregator
         end
       end
 
+      # Track the frequency of a specific category
+      # <tt>category_field</tt> Field to track
+      # <tt>options</tt> options are passed to new frequency tracker
+      def count(category_field, options = {})
+        if category_field.kind_of?(Symbol)
+          track(:count, options.merge(:category => category_field))
+        elsif category_field.kind_of?(Hash)
+          track(:count, category_field.merge(options))
+        end
+      end
+
       # Helper function to initialize a tracker and add it to the tracker array.
       # <tt>tracker_class</tt> The class to include
       # <tt>optiont</tt> The options to pass to the trackers.
@@ -90,7 +101,7 @@ module RequestLogAnalyzer::Aggregator
     # Call finalize on all trackers. Saves a YAML dump if this is set in  the options.
     def finalize
       @trackers.each { |tracker| tracker.finalize }
-      save_results_dump(options[:dump]) if options[:dump]
+      save_results_dump(options[:yaml]) if options[:yaml]
     end
 
     # Saves the results of all the trackers in YAML format to a file.
@@ -145,7 +156,7 @@ module RequestLogAnalyzer::Aggregator
       if has_log_ordering_warnings?
         output.title("Parse warnings")
 
-        output.puts "Parseable lines were ancountered without a header line before it. It"
+        output.puts "Parseable lines were encountered without a header line before it. It"
         output.puts "could be that logging is not setup correctly for your application."
         output.puts "Visit this website for logging configuration tips:"
         output.puts output.link("http://github.com/wvanbergen/request-log-analyzer/wikis/configure-logging")

data/lib/request_log_analyzer/controller.rb

@@ -1,14 +1,12 @@
 module RequestLogAnalyzer
 
   # The RequestLogAnalyzer::Controller class creates a LogParser instance for the
-  # requested file format, and connect it with sources and aggregators.
+  # requested file format and connect it with sources and aggregators.
   #
   # Sources are streams or files from which the requests will be parsed.
   # Aggregators will handle every passed request to yield a meaningfull results.
   #
-  # - Use the build-function to build a controller instance using command line arguments.
-  # - Use add_aggregator to register a new aggregator
-  # - Use add_source to register a new aggregator
+  # - Use the build -function to build a new controller instance.
   # - Use the run! method to start the parser and send the requests to the aggregators.
   #
   # Note that the order of sources can be imported if you have log files than succeed
@@ -21,7 +19,6 @@ module RequestLogAnalyzer
 
     # Builds a RequestLogAnalyzer::Controller given parsed command line arguments
     # <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
-    # <rr>report_with</tt> Width of the report. Defaults to 80.
     def self.build_from_arguments(arguments)
       
       options = {}
@@ -30,13 +27,12 @@ module RequestLogAnalyzer
       options[:database]       = arguments[:database]
       options[:reset_database] = arguments[:reset_database]
       options[:debug]          = arguments[:debug]
-      options[:dump]           = arguments[:dump]
-      options[:parse_strategy] = arguments[:parse_strategy]
+      options[:yaml]           = arguments[:yaml] || arguments[:dump]
+      options[:mail]           = arguments[:mail]
       options[:no_progress]    = arguments[:no_progress]
       options[:format]         = arguments[:format]
-      options[:output]         = arguments[:output]
+      options[:output]         = arguments[:output].downcase
       options[:file]           = arguments[:file]
-      options[:format]         = arguments[:format]
       options[:after]          = arguments[:after]
       options[:before]         = arguments[:before]
       options[:reject]         = arguments[:reject]
@@ -46,6 +42,7 @@ module RequestLogAnalyzer
       options[:report_width]   = arguments[:report_width]
       options[:report_sort]    = arguments[:report_sort]
       options[:report_amount]  = arguments[:report_amount]
+      options[:mailhost]       = arguments[:mailhost] 
       
       # Apache format workaround
       if arguments[:rails_format]
@@ -54,6 +51,12 @@ module RequestLogAnalyzer
         options[:format] = {:apache => arguments[:apache_format]}
       end
       
+      # Handle output format casing
+      if options[:output].class == String
+        options[:output] = 'HTML' if options[:output] == 'html'
+        options[:output] = 'FixedWidth' if options[:output] == 'fixedwidth' || options[:output] == 'fixed_width'
+      end
+      
       # Register sources
       if arguments.parameters.length == 1
         file = arguments.parameters[0]
@@ -69,38 +72,60 @@ module RequestLogAnalyzer
         options.store(:source_files, arguments.parameters)
       end
       
+      # Guess file format
+      if !options[:format] && options[:source_files]
+        options[:format] = :rails # Default
+
+        if options[:source_files] != $stdin
+          if options[:source_files].class == String
+            options[:format] = RequestLogAnalyzer::FileFormat.autodetect(options[:source_files])
+
+          elsif options[:source_files].class == Array && options[:source_files].first != $stdin
+            options[:format] = RequestLogAnalyzer::FileFormat.autodetect(options[:source_files].first)
+          end
+        end
+      end
+      
       build(options)
     end
 
-    # Build a new controller using parameters (Base for new API)
-    # <tt>source</tt> The source file
-    # Options are passd on to the LogParser.
+    # Build a new controller.
+    # Returns a new RequestLogAnalyzer::Controller object.
     #
     # Options
-    # * <tt>:database</tt> Database file
-    # * <tt>:reset_database</tt> 
-    # * <tt>:debug</tt> Enables echo aggregator.
-    # * <tt>:dump</tt> 
-    # * <tt>:parse_strategy</tt> 
-    # * <tt>:no_progress</tt> 
-    # * <tt>:output</tt> :fixed_width, :html or Output class. Defaults to fixed width.
-    # * <tt>:file</tt> Filestring or File or StringIO
-    # * <tt>:format</tt> :rails, {:apache => 'FORMATSTRING'}, :merb, etcetera or Format Class. Defaults to :rails.
-    # * <tt>:source_files</tt> File or STDIN
     # * <tt>:after</tt> Drop all requests after this date (Date, DateTime, Time, or a String in "YYYY-MM-DD hh:mm:ss" format)
-    # * <tt>:before</tt> Drop all requests before this date (Date, DateTime, Time, or a String in "YYYY-MM-DD hh:mm:ss" format)
-    # * <tt>:reject</tt> Reject specific {:field => :value} combination. Expects single hash.
-    # * <tt>:select</tt> Select specific {:field => :value} combination. Expects single hash.
-    # * <tt>:aggregator</tt> Array of aggregators (ATM: STRINGS OR SYMBOLS ONLY!). Defaults to [:summarizer
-    # * <tt>:boring</tt> Do not show color on STDOUT. Defaults to False.
-    # * <tt>:report_width</tt> Width or reports in characters. Defaults to 80.
+    # * <tt>:aggregator</tt> Array of aggregators (ATM: STRINGS OR SYMBOLS ONLY! - Defaults to [:summarizer]).
+    # * <tt>:boring</tt> Do not show color on STDOUT (Defaults to false).
+    # * <tt>:before</tt> Drop all requests before this date (Date, DateTime, Time or a String in "YYYY-MM-DD hh:mm:ss" format)
+    # * <tt>:database</tt> Database file to insert encountered requests to.
+    # * <tt>:debug</tt> Enables echo aggregator which will echo each request analyzed.
+    # * <tt>:file</tt> Filestring, File or StringIO.
+    # * <tt>:format</tt> :rails, {:apache => 'FORMATSTRING'}, :merb, :amazon_s3, :mysql or RequestLogAnalyzer::FileFormat class. (Defaults to :rails).
+    # * <tt>:mail</tt> Email the results to this email address.
+    # * <tt>:mailhost</tt> Email the results to this mail server.
+    # * <tt>:no_progress</tt> Do not display the progress bar (increases parsing speed).
+    # * <tt>:output</tt> 'FixedWidth', 'HTML' or RequestLogAnalyzer::Output class. Defaults to 'FixedWidth'.
+    # * <tt>:reject</tt> Reject specific {:field => :value} combination (expects a single hash).
+    # * <tt>:report_width</tt> Width of reports in characters for FixedWidth reports. (Defaults to 80)
+    # * <tt>:reset_database</tt> Reset the database before starting.
+    # * <tt>:select</tt> Select specific {:field => :value} combination (expects a single hash).
+    # * <tt>:source_files</tt> Source files to analyze. Provide either File, array of files or STDIN.
+    # * <tt>:yaml</tt> Output to YAML file.
     #
-    # TODO:
-    #   Check if defaults work (Aggregator defaults seem wrong).
-    #   Refactor :database => options[:database], :dump => options[:dump] away from contoller intialization.
+    # === Example
+    # RequestLogAnalyzer::Controller.build(
+    #   :output       => :HTML,
+    #   :mail         => 'root@localhost',
+    #   :after        => Time.now - 24*60*60, 
+    #   :source_files => '/var/log/passenger.log'
+    # ).run!
+    #
+    # === Todo
+    # * Check if defaults work (Aggregator defaults seem wrong).
+    # * Refactor :database => options[:database], :dump => options[:dump] away from contoller intialization.
     def self.build(options)
       # Defaults
-      options[:output]        ||= :fixed_width
+      options[:output]        ||= 'FixedWidth'
       options[:format]        ||= :rails
       options[:aggregator]    ||= [:summarizer]
       options[:report_width]  ||= 80
@@ -108,6 +133,12 @@ module RequestLogAnalyzer
       options[:report_sort]   ||= 'sum,mean'
       options[:boring]        ||= false
       
+      # Deprecation warnings
+      if options[:dump]
+        warn "[DEPRECATION] `:dump` is deprecated.  Please use `:yaml` instead."
+        options[:yaml]          = options[:dump]
+      end
+      
       # Set the output class
       output_args   = {}
       output_object = nil
@@ -124,7 +155,7 @@ module RequestLogAnalyzer
         output_object = %w[File StringIO].include?(options[:file].class.name) ? options[:file] : File.new(options[:file], "w+")
         output_args   = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount }
       elsif options[:mail]
-        output_object = RequestLogAnalyzer::Mailer.new(arguments[:mail])
+        output_object = RequestLogAnalyzer::Mailer.new(options[:mail], options[:mailhost])
         output_args   = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount  }
       else
         output_object = STDOUT
@@ -145,8 +176,9 @@ module RequestLogAnalyzer
       controller = Controller.new(  RequestLogAnalyzer::Source::LogParser.new(file_format, :source_files => options[:source_files]),
                                     { :output => output_instance,
                                       :database => options[:database],                # FUGLY!
-                                      :dump => options[:dump], 
-                                      :reset_database => options[:reset_database]})
+                                      :yaml => options[:yaml], 
+                                      :reset_database => options[:reset_database],
+                                      :no_progress => options[:no_progress]})
 
       # register filters
       if options[:after] || options[:before]
@@ -188,7 +220,7 @@ module RequestLogAnalyzer
     # <tt>format</tt> Logfile format. Defaults to :rails
     # Options are passd on to the LogParser.
     # * <tt>:database</tt> Database the controller should use.
-    # * <tt>:dump</tt> Yaml Dump the contrller should use.
+    # * <tt>:yaml</tt> Yaml Dump the contrller should use.
     # * <tt>:output</tt> All report outputs get << through this output.
     # * <tt>:no_progress</tt> No progress bar
     def initialize(source, options = {})

data/lib/request_log_analyzer/file_format/apache.rb

@@ -55,7 +55,7 @@ module RequestLogAnalyzer::FileFormat
 
     # Creates the access log line definition based on the Apache log format string
     def self.access_line_definition(format_string)
-      format_string ||= :combined
+      format_string ||= :common
       format_string   = LOG_FORMAT_DEFAULTS[format_string.to_sym] || format_string
 
       line_regexp = ''

data/lib/request_log_analyzer/file_format/mysql.rb

@@ -0,0 +1,93 @@
+module RequestLogAnalyzer::FileFormat
+
+  class Mysql < Base
+
+    line_definition :time do |line|
+      line.header = :alternative
+      line.teaser = /\# Time: /
+      line.regexp = /\# Time: (\d{6}\s+\d{1,2}:\d{2}:\d{2})/
+      line.captures << { :name => :timestamp, :type => :timestamp }
+    end
+
+    line_definition :user_host do |line|
+      line.header = :alternative
+      line.teaser = /\# User\@Host\: /
+      line.regexp = /\# User\@Host\: ([\w-]+)\[[\w-]+\] \@ ([\w\.-]*) \[([\d\.]*)\]/
+      line.captures << { :name => :user, :type => :string } << 
+                       { :name => :host, :type => :string } <<
+                       { :name => :ip,   :type => :string }
+    end
+
+    line_definition :query_statistics do |line|
+      line.header = :alternative
+      line.teaser = /\# Query_time: /
+      line.regexp = /\# Query_time: (\d+(?:\.\d+)?)\s+Lock_time: (\d+(?:\.\d+)?)\s+Rows_sent: (\d+)\s+Rows_examined: (\d+)/
+      line.captures << { :name => :query_time, :type => :duration, :unit => :sec } <<
+                       { :name => :lock_time,  :type => :duration, :unit => :sec } <<
+                       { :name => :rows_sent, :type => :integer } <<
+                       { :name => :rows_examined, :type => :integer }
+    end
+
+    line_definition :use_database do |line|
+      line.regexp   = /^use (\w+);\s*$/
+      line.captures << { :name => :database, :type => :string }
+    end
+
+    line_definition :query_part do |line|
+      line.regexp   = /^(?!(?:use |\# |SET ))(.*[^;\s])\s*$/
+      line.captures << { :name => :query_fragment, :type => :string }
+    end
+
+    line_definition :query do |line|
+      line.regexp = /^(?!(?:use |\# |SET ))(.*);\s*$/
+      line.captures << { :name => :query, :type => :sql }
+      line.footer = true
+    end
+
+    PER_USER       = :user
+    PER_QUERY      = :query
+    PER_USER_QUERY = Proc.new { |request| "#{request[:user]}@#{request.host}: #{request[:query]}" }
+
+    report do |analyze|
+      analyze.timespan :line_type => :time
+      analyze.frequency :user, :title => "Users with most queries"
+      analyze.duration :query_time, :category => PER_USER, :title => 'Query time per user'
+      analyze.duration :query_time, :category => PER_USER_QUERY, :title => 'Query time'
+      # => analyze.duration :lock_time,  :category => PER_USER_QUERY, :title => 'Lock time'
+      analyze.count :category => PER_USER_QUERY, :title => "Rows examined", :field => :rows_examined
+      analyze.count :category => PER_USER_QUERY, :title => "Rows sent",     :field => :rows_sent
+    end
+  
+    class Request < RequestLogAnalyzer::Request
+
+      def convert_sql(value, definition)
+
+        # Recreate the full SQL query by joining all the previous parts and this last line
+        sql = every(:query_fragment).join("\n") + value
+
+        # Sanitize an SQL query so that it can be used as a category field.
+        # sql.gsub!(/\/\*.*\*\//, '')                                       # remove comments
+        sql.gsub!(/\s+/, ' ')                                             # remove excessive whitespace
+        sql.gsub!(/`([^`]+)`/, '\1')                                      # remove quotes from field names
+        sql.gsub!(/'\d{4}-\d{2}-\d{2}'/, ':date')                         # replace dates
+        sql.gsub!(/'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}'/, ':datetime')   # replace timestamps
+        sql.gsub!(/'[^']*'/, ':string')                                   # replace strings
+        sql.gsub!(/\b\d+\b/, ':int')                                      # replace integers
+        sql.gsub!(/(:int,)+:int/, ':ints')                                # replace multiple ints by a list
+        sql.gsub!(/(:string,)+:string/, ':strings')                       # replace multiple strings by a list
+
+        return sql.rstrip
+      end
+
+      def host
+        self[:host] == '' || self[:host].nil? ? self[:ip] : self[:host]
+      end
+
+      # Convert the timestamp to an integer
+      def convert_timestamp(value, definition)
+        all,y,m,d,h,i,s = value.split(/(\d\d)(\d\d)(\d\d)\s+(\d?\d):(\d\d):(\d\d)/)
+        ('20%s%s%s%s%s%s' % [y,m,d,h.rjust(2, '0'),i,s]).to_i
+      end
+    end
+  end
+end

data/lib/request_log_analyzer/file_format/rails_development.rb

@@ -5,7 +5,7 @@ module RequestLogAnalyzer::FileFormat
   # as well.
   class RailsDevelopment < Rails
     def self.create
-      puts 'DEPRECATED: use --rails-format development instead!'
+      # puts 'DEPRECATED: use --rails-format development instead!'
       super('development')
     end
   end

data/lib/request_log_analyzer/file_format.rb

@@ -1,6 +1,6 @@
 module RequestLogAnalyzer::FileFormat
 
-  def self.const_missing(const)
+  def self.const_missing(const) # :nodoc:
     RequestLogAnalyzer::load_default_class_file(self, const)
   end
 
@@ -39,10 +39,54 @@ module RequestLogAnalyzer::FileFormat
 
     # check the returned klass to see if it can be used
     raise "Could not load a file format from #{file_format.inspect}" if klass.nil?
-    raise "Invalid FileFormat class" unless klass.kind_of?(Class) && klass.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
+    raise "Invalid FileFormat class from #{file_format.inspect}" unless klass.kind_of?(Class) && klass.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
 
     @current_file_format = klass.create(*args) # return an instance of the class
   end
+  
+  # Returns an array of all FileFormat instances that are shipped with request-log-analyzer by default.
+  def self.all_formats
+    @all_formats ||= Dir[File.dirname(__FILE__) + '/file_format/*.rb'].map do |file| 
+      self.load(File.basename(file, '.rb')) 
+    end
+  end
+  
+  # Autodetects the filetype of a given file.
+  #
+  # Returns a FileFormat instance, by parsing the first couple of lines of the provided file
+  # with avery known file format and return the most promosing file format based on the parser
+  # statistics. The <tt>autodetect_score</tt> method is used to score the fitness of a format.
+  #
+  # <tt>file</tt>:: The file to detect the file format for.
+  # <tt>line_count</tt>:: The number of lines to take into consideration
+  def self.autodetect(file, line_count = 50)
+    
+    parsers = all_formats.map { |f| RequestLogAnalyzer::Source::LogParser.new(f, :parse_strategy => 'cautious') }
+    
+    File.open(file, 'r') do |io|
+      while io.lineno < line_count && (line = io.gets)
+        parsers.each { |parser| parser.parse_line(line) } 
+      end
+    end
+    
+    parsers.select { |p| autodetect_score(p) > 0 }.max { |a, b| autodetect_score(a) <=> autodetect_score(b) }.file_format rescue nil
+  end
+  
+  # Calculates a file format auto detection score based on the parser statistics.
+  #
+  # This method returns a score as an integer. Usually, the score will increase as more
+  # lines are parsed. Usually, a file_format with a score of zero or lower should not be
+  # considered.
+  #
+  # <tt>parser</tt>:: The parsed that was use to parse the initial lines of the log file.
+  def self.autodetect_score(parser)
+    score  = 0
+    score -= parser.file_format.line_definitions.length
+    score -= parser.warnings * 3
+    score += parser.parsed_lines * 1
+    score += parser.parsed_requests * 10
+    score
+  end
 
   # Base class for all log file format definitions. This class provides functions for subclasses to
   # define their LineDefinitions and to define a summary report.

data/lib/request_log_analyzer/mailer.rb

@@ -1,38 +1,51 @@
 module RequestLogAnalyzer
 
+  # Mail report to a specified emailaddress
   class Mailer
 
-    attr_accessor :data, :to, :host
+    attr_accessor :data, :to, :host, :port
 
     # Initialize a mailer
-    # <tt>to</tt> to address
-    # <tt>host</tt> the mailer host
+    # <tt>to</tt> to email address to mail to
+    # <tt>host</tt> the mailer host (defaults to localhost)
     # <tt>options</tt> Specific style options
+    #
     # Options
-    # <tt>:debug</tt> Do not actually mail
+    # * <tt>:debug</tt> Do not actually mail
+    # * <tt>:from_alias</tt> The from alias
+    # * <tt>:to_alias</tt> The to alias
+    # * <tt>:subject</tt> The message subject
     def initialize(to, host = 'localhost', options = {})
       require 'net/smtp'
       @to      = to
       @host    = host
+
+      @port    = 25
       @options = options
-      @data    = []
+      @host, @port = host.split(':') if @host.include?(':')
+      @data    = []      
     end
 
+    # Send all data in @data to the email address used during initialization.
+    # Returns array containg [message_data, from_email_address, to_email_address] of sent email.
     def mail
-      from        = @options[:from]        || 'contact@railsdoctors.com'
-      from_alias  = @options[:from_alias]  || 'Request-log-analyzer reporter'
-      to_alias    = @options[:to_alias]    || to
-      subject     = @options[:subjeect]    || "Request log analyzer report - generated on #{Time.now.to_s}"
+      from          = @options[:from]        || 'contact@railsdoctors.com'
+      from_alias    = @options[:from_alias]  || 'Request-log-analyzer reporter'
+      to_alias      = @options[:to_alias]    || to
+      subject       = @options[:subject]     || "Request log analyzer report - generated on #{Time.now.to_s}"
+      content_type  = ""
+      content_type  = 'Content-Type: text/html; charset="ISO-8859-1";' if @data.map{|l| l.include?('html')}.include?(true)
     msg = <<END_OF_MESSAGE
 From: #{from_alias} <#{from}>
 To: #{to_alias} <#{@to}>
 Subject: #{subject}
+#{content_type}
 
-#{@data.to_s}
+#{@data.join("\n")}
 END_OF_MESSAGE
-    
+
       unless @options[:debug]
-        Net::SMTP.start(@host) do |smtp|
+        Net::SMTP.start(@host, @port) do |smtp|
           smtp.send_message msg, from, to
         end
       end

data/lib/request_log_analyzer/output/fixed_width.rb

@@ -108,7 +108,7 @@ module RequestLogAnalyzer::Output
     # <tt>url</tt> The url to link to.
     def link(text, url = nil)
       if url.nil?
-        colorize(text, :blue, :bold)
+        colorize(text, :red, :bold)
       else
         "#{text} (#{colorize(url, :blue, :bold)})"
       end

data/lib/request_log_analyzer/output/html.rb

@@ -69,6 +69,8 @@ module RequestLogAnalyzer::Output
 
     # Genrate HTML header and associated stylesheet
     def header
+      @io.content_type = content_type if @io.respond_to?(:content_type)
+
       @io << "<html>"
       @io << tag(:head) do |headers|
         headers << tag(:title, 'Request-log-analyzer report')

data/lib/request_log_analyzer/request.rb

@@ -65,6 +65,11 @@ module RequestLogAnalyzer
         else raise "Unknown duration unit"
         end
       end
+      
+      # Convert an epoch to an integer
+      def convert_epoch(value, capture_definition)
+        Time.at(value.to_i).strftime('%Y%m%d%H%M%S').to_i
+      end
     end
 
     # Install the default converter methods