request-log-analyzer 1.4.2 → 1.5.0

data/bin/request-log-analyzer

@@ -27,14 +27,16 @@ begin
       strip.switch(:keep_junk_lines, :j)
     end
 
-    command_line.option(:format, :alias => :f, :default => 'rails')
+    command_line.option(:format, :alias => :f)
     command_line.option(:apache_format)
     command_line.option(:rails_format)
 
     command_line.option(:file, :alias => :e)
     command_line.option(:mail, :alias => :m)
+    command_line.option(:mailhost, :default => 'localhost')
     command_line.option(:parse_strategy, :default => 'assume-correct')
-    command_line.option(:dump)
+    command_line.option(:yaml)
+    command_line.option(:dump) # To be deprecated
 
     command_line.option(:aggregator, :alias => :a, :multiple => true)
 
@@ -48,7 +50,7 @@ begin
     command_line.option(:before)
 
     command_line.switch(:boring, :b)
-    command_line.option(:output, :alias => :o, :default => 'FixedWidth')
+    command_line.option(:output, :alias => :o, :default => 'fixedwidth')
     command_line.option(:report_width,  :default => terminal_width - 1)
     command_line.option(:report_amount, :default => 20)
     command_line.option(:report_sort,   :default => 'sum,mean')
@@ -68,7 +70,7 @@ rescue CommandLine::Error => e
   puts "Usage: request-log-analyzer [LOGFILES*] <OPTIONS>"
   puts
   puts "Input options:"
-  puts "  --format <format>, -f:     Uses the specified log file format. Defaults to rails."
+  puts "  --format <format>, -f:     Log file format. amazon_s3, apache, merb, mysql, rack or rails (default)."
   puts "  --after <date>             Only consider requests from <date> or later."
   puts "  --before <date>            Only consider requests before <date>."
   puts "  --select <field> <value>   Only consider requests where <field> matches <value>."
@@ -78,10 +80,11 @@ rescue CommandLine::Error => e
   puts "  --boring, -b               Output reports without ASCII colors."
   puts "  --database <filename>, -d: Creates an SQLite3 database of all the parsed request information."
   puts "  --debug                    Print debug information while parsing."
-  puts "  --file <filename>          Output to file."
+  puts "  --file <filename>          Redirect output to file."
   puts "  --mail <emailaddress>      Send report to an email address."
-  puts "  --output <format>          Output format. Supports 'HTML' and 'FixedWidth' (default)"
-  puts "  --dump <filename>          Dump the YAML formatted results in the given file"
+  puts "  --mailhost <server>        Use the given server as the SMTP server for sending email."
+  puts "  --output <format>          Output format. Supports 'html' and 'fixed_width' (default)."
+  puts "  --yaml <filename>          Dump the results in YAML format in the given file."
   puts
   puts "Examples:"
   puts "  request-log-analyzer development.log"

data/lib/request_log_analyzer.rb

@@ -11,7 +11,7 @@ module RequestLogAnalyzer
 
   # The current version of request-log-analyzer.
   # Do not change the value by hand; it will be updated automatically by the gem release script.
-  VERSION = "1.4.2"
+  VERSION = "1.5.0"
 
   # Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
   # <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.

data/lib/request_log_analyzer/aggregator/summarizer.rb

@@ -49,6 +49,17 @@ module RequestLogAnalyzer::Aggregator
         end
       end
 
+      # Track the frequency of a specific category
+      # <tt>category_field</tt> Field to track
+      # <tt>options</tt> options are passed to new frequency tracker
+      def count(category_field, options = {})
+        if category_field.kind_of?(Symbol)
+          track(:count, options.merge(:category => category_field))
+        elsif category_field.kind_of?(Hash)
+          track(:count, category_field.merge(options))
+        end
+      end
+
       # Helper function to initialize a tracker and add it to the tracker array.
       # <tt>tracker_class</tt> The class to include
       # <tt>optiont</tt> The options to pass to the trackers.
@@ -145,7 +156,7 @@ module RequestLogAnalyzer::Aggregator
       if has_log_ordering_warnings?
         output.title("Parse warnings")
 
-        output.puts "Parseable lines were ancountered without a header line before it. It"
+        output.puts "Parseable lines were encountered without a header line before it. It"
         output.puts "could be that logging is not setup correctly for your application."
         output.puts "Visit this website for logging configuration tips:"
         output.puts output.link("http://github.com/wvanbergen/request-log-analyzer/wikis/configure-logging")

data/lib/request_log_analyzer/controller.rb

@@ -27,13 +27,12 @@ module RequestLogAnalyzer
       options[:database]       = arguments[:database]
       options[:reset_database] = arguments[:reset_database]
       options[:debug]          = arguments[:debug]
-      options[:yaml]           = arguments[:dump]
+      options[:yaml]           = arguments[:yaml] || arguments[:dump]
       options[:mail]           = arguments[:mail]
       options[:no_progress]    = arguments[:no_progress]
       options[:format]         = arguments[:format]
-      options[:output]         = arguments[:output]
+      options[:output]         = arguments[:output].downcase
       options[:file]           = arguments[:file]
-      options[:format]         = arguments[:format]
       options[:after]          = arguments[:after]
       options[:before]         = arguments[:before]
       options[:reject]         = arguments[:reject]
@@ -43,6 +42,7 @@ module RequestLogAnalyzer
       options[:report_width]   = arguments[:report_width]
       options[:report_sort]    = arguments[:report_sort]
       options[:report_amount]  = arguments[:report_amount]
+      options[:mailhost]       = arguments[:mailhost] 
       
       # Apache format workaround
       if arguments[:rails_format]
@@ -51,6 +51,12 @@ module RequestLogAnalyzer
         options[:format] = {:apache => arguments[:apache_format]}
       end
       
+      # Handle output format casing
+      if options[:output].class == String
+        options[:output] = 'HTML' if options[:output] == 'html'
+        options[:output] = 'FixedWidth' if options[:output] == 'fixedwidth' || options[:output] == 'fixed_width'
+      end
+      
       # Register sources
       if arguments.parameters.length == 1
         file = arguments.parameters[0]
@@ -66,6 +72,20 @@ module RequestLogAnalyzer
         options.store(:source_files, arguments.parameters)
       end
       
+      # Guess file format
+      if !options[:format] && options[:source_files]
+        options[:format] = :rails # Default
+
+        if options[:source_files] != $stdin
+          if options[:source_files].class == String
+            options[:format] = RequestLogAnalyzer::FileFormat.autodetect(options[:source_files])
+
+          elsif options[:source_files].class == Array && options[:source_files].first != $stdin
+            options[:format] = RequestLogAnalyzer::FileFormat.autodetect(options[:source_files].first)
+          end
+        end
+      end
+      
       build(options)
     end
 
@@ -80,12 +100,13 @@ module RequestLogAnalyzer
     # * <tt>:database</tt> Database file to insert encountered requests to.
     # * <tt>:debug</tt> Enables echo aggregator which will echo each request analyzed.
     # * <tt>:file</tt> Filestring, File or StringIO.
-    # * <tt>:format</tt> :rails, {:apache => 'FORMATSTRING'}, :merb, etcetera or Format Class. (Defaults to :rails).
+    # * <tt>:format</tt> :rails, {:apache => 'FORMATSTRING'}, :merb, :amazon_s3, :mysql or RequestLogAnalyzer::FileFormat class. (Defaults to :rails).
     # * <tt>:mail</tt> Email the results to this email address.
-    # * <tt>:no_progress</tt> Do not display the progress bar (increases speed).
-    # * <tt>:output</tt> :fixed_width, :html or Output class. Defaults to fixed width.
+    # * <tt>:mailhost</tt> Email the results to this mail server.
+    # * <tt>:no_progress</tt> Do not display the progress bar (increases parsing speed).
+    # * <tt>:output</tt> 'FixedWidth', 'HTML' or RequestLogAnalyzer::Output class. Defaults to 'FixedWidth'.
     # * <tt>:reject</tt> Reject specific {:field => :value} combination (expects a single hash).
-    # * <tt>:report_width</tt> Width or reports in characters. (Defaults to 80)
+    # * <tt>:report_width</tt> Width of reports in characters for FixedWidth reports. (Defaults to 80)
     # * <tt>:reset_database</tt> Reset the database before starting.
     # * <tt>:select</tt> Select specific {:field => :value} combination (expects a single hash).
     # * <tt>:source_files</tt> Source files to analyze. Provide either File, array of files or STDIN.
@@ -104,7 +125,7 @@ module RequestLogAnalyzer
     # * Refactor :database => options[:database], :dump => options[:dump] away from contoller intialization.
     def self.build(options)
       # Defaults
-      options[:output]        ||= 'fixed_width'
+      options[:output]        ||= 'FixedWidth'
       options[:format]        ||= :rails
       options[:aggregator]    ||= [:summarizer]
       options[:report_width]  ||= 80
@@ -113,7 +134,7 @@ module RequestLogAnalyzer
       options[:boring]        ||= false
       
       # Deprecation warnings
-      if options[:dump] && options[:yaml].blank?
+      if options[:dump]
         warn "[DEPRECATION] `:dump` is deprecated.  Please use `:yaml` instead."
         options[:yaml]          = options[:dump]
       end
@@ -134,7 +155,7 @@ module RequestLogAnalyzer
         output_object = %w[File StringIO].include?(options[:file].class.name) ? options[:file] : File.new(options[:file], "w+")
         output_args   = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount }
       elsif options[:mail]
-        output_object = RequestLogAnalyzer::Mailer.new(options[:mail])
+        output_object = RequestLogAnalyzer::Mailer.new(options[:mail], options[:mailhost])
         output_args   = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount  }
       else
         output_object = STDOUT

data/lib/request_log_analyzer/file_format.rb

@@ -1,6 +1,6 @@
 module RequestLogAnalyzer::FileFormat
 
-  def self.const_missing(const)
+  def self.const_missing(const) # :nodoc:
     RequestLogAnalyzer::load_default_class_file(self, const)
   end
 
@@ -39,10 +39,54 @@ module RequestLogAnalyzer::FileFormat
 
     # check the returned klass to see if it can be used
     raise "Could not load a file format from #{file_format.inspect}" if klass.nil?
-    raise "Invalid FileFormat class" unless klass.kind_of?(Class) && klass.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
+    raise "Invalid FileFormat class from #{file_format.inspect}" unless klass.kind_of?(Class) && klass.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
 
     @current_file_format = klass.create(*args) # return an instance of the class
   end
+  
+  # Returns an array of all FileFormat instances that are shipped with request-log-analyzer by default.
+  def self.all_formats
+    @all_formats ||= Dir[File.dirname(__FILE__) + '/file_format/*.rb'].map do |file| 
+      self.load(File.basename(file, '.rb')) 
+    end
+  end
+  
+  # Autodetects the filetype of a given file.
+  #
+  # Returns a FileFormat instance, by parsing the first couple of lines of the provided file
+  # with avery known file format and return the most promosing file format based on the parser
+  # statistics. The <tt>autodetect_score</tt> method is used to score the fitness of a format.
+  #
+  # <tt>file</tt>:: The file to detect the file format for.
+  # <tt>line_count</tt>:: The number of lines to take into consideration
+  def self.autodetect(file, line_count = 50)
+    
+    parsers = all_formats.map { |f| RequestLogAnalyzer::Source::LogParser.new(f, :parse_strategy => 'cautious') }
+    
+    File.open(file, 'r') do |io|
+      while io.lineno < line_count && (line = io.gets)
+        parsers.each { |parser| parser.parse_line(line) } 
+      end
+    end
+    
+    parsers.select { |p| autodetect_score(p) > 0 }.max { |a, b| autodetect_score(a) <=> autodetect_score(b) }.file_format rescue nil
+  end
+  
+  # Calculates a file format auto detection score based on the parser statistics.
+  #
+  # This method returns a score as an integer. Usually, the score will increase as more
+  # lines are parsed. Usually, a file_format with a score of zero or lower should not be
+  # considered.
+  #
+  # <tt>parser</tt>:: The parsed that was use to parse the initial lines of the log file.
+  def self.autodetect_score(parser)
+    score  = 0
+    score -= parser.file_format.line_definitions.length
+    score -= parser.warnings * 3
+    score += parser.parsed_lines * 1
+    score += parser.parsed_requests * 10
+    score
+  end
 
   # Base class for all log file format definitions. This class provides functions for subclasses to
   # define their LineDefinitions and to define a summary report.

data/lib/request_log_analyzer/file_format/apache.rb

@@ -55,7 +55,7 @@ module RequestLogAnalyzer::FileFormat
 
     # Creates the access log line definition based on the Apache log format string
     def self.access_line_definition(format_string)
-      format_string ||= :combined
+      format_string ||= :common
       format_string   = LOG_FORMAT_DEFAULTS[format_string.to_sym] || format_string
 
       line_regexp = ''

data/lib/request_log_analyzer/file_format/mysql.rb

@@ -0,0 +1,93 @@
+module RequestLogAnalyzer::FileFormat
+
+  class Mysql < Base
+
+    line_definition :time do |line|
+      line.header = :alternative
+      line.teaser = /\# Time: /
+      line.regexp = /\# Time: (\d{6}\s+\d{1,2}:\d{2}:\d{2})/
+      line.captures << { :name => :timestamp, :type => :timestamp }
+    end
+
+    line_definition :user_host do |line|
+      line.header = :alternative
+      line.teaser = /\# User\@Host\: /
+      line.regexp = /\# User\@Host\: ([\w-]+)\[[\w-]+\] \@ ([\w\.-]*) \[([\d\.]*)\]/
+      line.captures << { :name => :user, :type => :string } << 
+                       { :name => :host, :type => :string } <<
+                       { :name => :ip,   :type => :string }
+    end
+
+    line_definition :query_statistics do |line|
+      line.header = :alternative
+      line.teaser = /\# Query_time: /
+      line.regexp = /\# Query_time: (\d+(?:\.\d+)?)\s+Lock_time: (\d+(?:\.\d+)?)\s+Rows_sent: (\d+)\s+Rows_examined: (\d+)/
+      line.captures << { :name => :query_time, :type => :duration, :unit => :sec } <<
+                       { :name => :lock_time,  :type => :duration, :unit => :sec } <<
+                       { :name => :rows_sent, :type => :integer } <<
+                       { :name => :rows_examined, :type => :integer }
+    end
+
+    line_definition :use_database do |line|
+      line.regexp   = /^use (\w+);\s*$/
+      line.captures << { :name => :database, :type => :string }
+    end
+
+    line_definition :query_part do |line|
+      line.regexp   = /^(?!(?:use |\# |SET ))(.*[^;\s])\s*$/
+      line.captures << { :name => :query_fragment, :type => :string }
+    end
+
+    line_definition :query do |line|
+      line.regexp = /^(?!(?:use |\# |SET ))(.*);\s*$/
+      line.captures << { :name => :query, :type => :sql }
+      line.footer = true
+    end
+
+    PER_USER       = :user
+    PER_QUERY      = :query
+    PER_USER_QUERY = Proc.new { |request| "#{request[:user]}@#{request.host}: #{request[:query]}" }
+
+    report do |analyze|
+      analyze.timespan :line_type => :time
+      analyze.frequency :user, :title => "Users with most queries"
+      analyze.duration :query_time, :category => PER_USER, :title => 'Query time per user'
+      analyze.duration :query_time, :category => PER_USER_QUERY, :title => 'Query time'
+      # => analyze.duration :lock_time,  :category => PER_USER_QUERY, :title => 'Lock time'
+      analyze.count :category => PER_USER_QUERY, :title => "Rows examined", :field => :rows_examined
+      analyze.count :category => PER_USER_QUERY, :title => "Rows sent",     :field => :rows_sent
+    end
+  
+    class Request < RequestLogAnalyzer::Request
+
+      def convert_sql(value, definition)
+
+        # Recreate the full SQL query by joining all the previous parts and this last line
+        sql = every(:query_fragment).join("\n") + value
+
+        # Sanitize an SQL query so that it can be used as a category field.
+        # sql.gsub!(/\/\*.*\*\//, '')                                       # remove comments
+        sql.gsub!(/\s+/, ' ')                                             # remove excessive whitespace
+        sql.gsub!(/`([^`]+)`/, '\1')                                      # remove quotes from field names
+        sql.gsub!(/'\d{4}-\d{2}-\d{2}'/, ':date')                         # replace dates
+        sql.gsub!(/'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}'/, ':datetime')   # replace timestamps
+        sql.gsub!(/'[^']*'/, ':string')                                   # replace strings
+        sql.gsub!(/\b\d+\b/, ':int')                                      # replace integers
+        sql.gsub!(/(:int,)+:int/, ':ints')                                # replace multiple ints by a list
+        sql.gsub!(/(:string,)+:string/, ':strings')                       # replace multiple strings by a list
+
+        return sql.rstrip
+      end
+
+      def host
+        self[:host] == '' || self[:host].nil? ? self[:ip] : self[:host]
+      end
+
+      # Convert the timestamp to an integer
+      def convert_timestamp(value, definition)
+        all,y,m,d,h,i,s = value.split(/(\d\d)(\d\d)(\d\d)\s+(\d?\d):(\d\d):(\d\d)/)
+        ('20%s%s%s%s%s%s' % [y,m,d,h.rjust(2, '0'),i,s]).to_i
+      end
+    end
+  end
+end

data/lib/request_log_analyzer/file_format/rails_development.rb

@@ -5,7 +5,7 @@ module RequestLogAnalyzer::FileFormat
   # as well.
   class RailsDevelopment < Rails
     def self.create
-      puts 'DEPRECATED: use --rails-format development instead!'
+      # puts 'DEPRECATED: use --rails-format development instead!'
       super('development')
     end
   end

data/lib/request_log_analyzer/mailer.rb

@@ -3,7 +3,7 @@ module RequestLogAnalyzer
   # Mail report to a specified emailaddress
   class Mailer
 
-    attr_accessor :data, :to, :host
+    attr_accessor :data, :to, :host, :port
 
     # Initialize a mailer
     # <tt>to</tt> to email address to mail to
@@ -19,8 +19,11 @@ module RequestLogAnalyzer
       require 'net/smtp'
       @to      = to
       @host    = host
+
+      @port    = 25
       @options = options
-      @data    = []
+      @host, @port = host.split(':') if @host.include?(':')
+      @data    = []      
     end
 
     # Send all data in @data to the email address used during initialization.
@@ -38,11 +41,11 @@ To: #{to_alias} <#{@to}>
 Subject: #{subject}
 #{content_type}
 
-#{@data.to_s}
+#{@data.join("\n")}
 END_OF_MESSAGE
 
       unless @options[:debug]
-        Net::SMTP.start(@host) do |smtp|
+        Net::SMTP.start(@host, @port) do |smtp|
           smtp.send_message msg, from, to
         end
       end

data/lib/request_log_analyzer/output/fixed_width.rb

@@ -108,7 +108,7 @@ module RequestLogAnalyzer::Output
     # <tt>url</tt> The url to link to.
     def link(text, url = nil)
       if url.nil?
-        colorize(text, :blue, :bold)
+        colorize(text, :red, :bold)
       else
         "#{text} (#{colorize(url, :blue, :bold)})"
       end

data/lib/request_log_analyzer/request.rb

@@ -65,6 +65,11 @@ module RequestLogAnalyzer
         else raise "Unknown duration unit"
         end
       end
+      
+      # Convert an epoch to an integer
+      def convert_epoch(value, capture_definition)
+        Time.at(value.to_i).strftime('%Y%m%d%H%M%S').to_i
+      end
     end
 
     # Install the default converter methods

data/lib/request_log_analyzer/source/log_parser.rb

@@ -20,6 +20,7 @@ module RequestLogAnalyzer::Source
     PARSE_STRATEGIES = ['cautious', 'assume-correct']
 
     attr_reader :source_files, :current_file, :current_lineno
+    attr_reader :warnings, :parsed_lines, :parsed_requests, :skipped_lines, :skipped_requests
 
     # Initializes the log file parser instance.
     # It will apply the language specific FileFormat module to this instance. It will use the line
@@ -29,6 +30,7 @@ module RequestLogAnalyzer::Source
     # <tt>options</tt>:: A hash of options that are used by the parser
     def initialize(format, options = {})
       super(format, options)
+      @warnings         = 0
       @parsed_lines     = 0
       @parsed_requests  = 0
       @skipped_lines    = 0
@@ -142,21 +144,27 @@ module RequestLogAnalyzer::Source
     # <tt>io</tt>:: The IO instance to use as source
     # <tt>options</tt>:: A hash of options that can be used by the parser.
     def parse_io(io, options = {}, &block) # :yields: request
-      @current_lineno = 1
+      @current_lineno = 0
       while line = io.gets
-        @progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno % 255 == 0
-
-        if request_data = file_format.parse_line(line) { |wt, message| warn(wt, message) }
-          @parsed_lines += 1
-          update_current_request(request_data.merge(:source => @current_source, :lineno => @current_lineno), &block)
-        end
-
         @current_lineno += 1
+        @progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno % 255 == 0
+        parse_line(line, &block)
       end
 
       warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
       @current_lineno = nil
     end
+    
+    # Parses a single line using the current file format. If successful, use the parsed
+    # information to build a request
+    # <tt>line</tt>:: The line to parse
+    # <tt>block</tt>:: The block to send fully parsed requests to.
+    def parse_line(line, &block) # :yields: request
+      if request_data = file_format.parse_line(line) { |wt, message| warn(wt, message) }
+        @parsed_lines += 1
+        update_current_request(request_data.merge(:source => @current_source, :lineno => @current_lineno), &block)
+      end
+    end
 
     # Add a block to this method to install a progress handler while parsing.
     # <tt>proc</tt>:: The proc that will be called to handle progress update messages
@@ -186,6 +194,7 @@ module RequestLogAnalyzer::Source
     # <tt>type</tt>:: The warning type (a Symbol)
     # <tt>message</tt>:: A message explaining the warning
     def warn(type, message)
+      @warnings += 1
       @warning_handler.call(type, message, @current_lineno) if @warning_handler
     end
 
@@ -214,7 +223,13 @@ module RequestLogAnalyzer::Source
     #
     # <tt>request_data</tt>:: A hash of data that was parsed from the last line.
     def update_current_request(request_data, &block) # :yields: request
-      if header_line?(request_data)
+      if alternative_header_line?(request_data)
+        if @current_request
+          @current_request << request_data
+        else
+          @current_request = @file_format.request(request_data)
+        end
+      elsif header_line?(request_data)
         if @current_request
           case options[:parse_strategy]
           when 'assume-correct'
@@ -258,10 +273,18 @@ module RequestLogAnalyzer::Source
       @skipped_requests += 1 unless accepted
     end
 
+
+    # Checks whether a given line hash is an alternative header line according to the current file format.
+    # <tt>hash</tt>:: A hash of data that was parsed from the line.
+    def alternative_header_line?(hash)
+      hash[:line_definition].header == :alternative
+    end
+
+
     # Checks whether a given line hash is a header line according to the current file format.
     # <tt>hash</tt>:: A hash of data that was parsed from the line.
     def header_line?(hash)
-      hash[:line_definition].header
+      hash[:line_definition].header == true
     end
 
     # Checks whether a given line hash is a footer line  according to the current file format.