request-log-analyzer 1.3.5 → 1.4.0

data/lib/request_log_analyzer/output.rb

@@ -5,13 +5,49 @@ module RequestLogAnalyzer::Output
   def self.const_missing(const)
     RequestLogAnalyzer::load_default_class_file(self, const)
   end
+
+  # Loads a Output::Base subclass instance.
+  def self.load(file_format, *args)
+    
+    klass = nil
+    if file_format.kind_of?(RequestLogAnalyzer::Output::Base)
+      # this already is a file format! return itself
+      return file_format
+
+    elsif file_format.kind_of?(Class) && file_format.ancestors.include?(RequestLogAnalyzer::Output::Base)
+      # a usable class is provided. Use this format class.
+      klass = file_format
+
+    elsif file_format.kind_of?(String) && File.exist?(file_format)
+      # load a format from a ruby file
+      require file_format
+      const = RequestLogAnalyzer::to_camelcase(File.basename(file_format, '.rb'))
+      if RequestLogAnalyzer::FileFormat.const_defined?(const)
+        klass = RequestLogAnalyzer::Output.const_get(const)
+      elsif Object.const_defined?(const)
+        klass = Object.const_get(const)
+      else
+        raise "Cannot load class #{const} from #{file_format}!"
+      end
+
+    else
+      # load a provided file format
+      klass = RequestLogAnalyzer::Output.const_get(RequestLogAnalyzer::to_camelcase(file_format))
+    end
+
+    # check the returned klass to see if it can be used
+    raise "Could not load a file format from #{file_format.inspect}" if klass.nil?
+    raise "Invalid FileFormat class" unless klass.kind_of?(Class) && klass.ancestors.include?(RequestLogAnalyzer::Output::Base)
+
+    klass.create(*args) # return an instance of the class
+  end
   
   # Base Class used for generating output for reports.
   # All output should inherit fromt this class.
   class Base
-    
+
     attr_accessor :io, :options, :style
-    
+
     # Initialize a report
     # <tt>io</tt> iO Object (file, STDOUT, etc.)
     # <tt>options</tt> Specific style options
@@ -27,32 +63,37 @@ module RequestLogAnalyzer::Output
       @style = @style.merge(temp_style)
       yield(self) if block_given?
       @style = old_style
-    end    
-    
+    end
+
     # Generate a header for a report
     def header
     end
-    
-    # Generate the footer of a report 
+
+    # Generate the footer of a report
     def footer
     end
 
+    def slice_results(array)
+      return array if options[:amount] == :all
+      return array.slice(0, options[:amount]) # otherwise
+    end
+
     # Generate a report table and push it into the output object.
     # Yeilds a rows array into which the rows can be pushed
     # <tt>*colums<tt> Array of Column hashes (see Column options).
     # <tt>&block</tt>: A block yeilding the rows.
-    # 
+    #
     # === Column options
     # Columns is an array of hashes containing the column definitions.
     # * <tt>:align</tt> Alignment :left or :right
     # * <tt>:treshold</tt> Width in characters or :rest
     # * <tt>:type</tt> :ratio or nil
     # * <tt>:width</tt> Width in characters or :rest
-    # 
+    #
     # === Example
     # The output object should support table definitions:
     #
-    # output.table({:align => :left}, {:align => :right }, {:align => :right}, {:type => :ratio, :width => :rest}) do |rows|        
+    # output.table({:align => :left}, {:align => :right }, {:align => :right}, {:type => :ratio, :width => :rest}) do |rows|
     #   sorted_frequencies.each do |(cat, count)|
     #     rows << [cat, "#{count} hits", '%0.1f%%' % ((count.to_f / total_hits.to_f) * 100.0), (count.to_f / total_hits.to_f)]
     #   end
@@ -60,13 +101,13 @@ module RequestLogAnalyzer::Output
     #
     def table(*columns, &block)
     end
-    
+
     protected
     # Check if a given table defination hash includes a header (title)
     # <tt>columns</tt> The columns hash
     def table_has_header?(columns)
-      columns.any? { |column| !column[:title].nil? } 
+      columns.any? { |column| !column[:title].nil? }
     end
-    
+
   end
 end

data/lib/request_log_analyzer/request.rb

@@ -1,83 +1,85 @@
 module RequestLogAnalyzer
-  
-  # The Request class represents a parsed request from the log file. 
+
+  # The Request class represents a parsed request from the log file.
   # Instances are created by the LogParser and are passed to the different aggregators, so they
-  # can do their aggregating work. 
+  # can do their aggregating work.
   #
   # This class provides several methods to access the data that was parsed from the log files.
   # Request#first(field_name) returns the first (only) value corresponding to the given field
   # Request#every(field_name) returns all values corresponding to the given field name as array.
   class Request
-  
+
     module Converters
-      
+
+      # Default converter function, which converts the parsed strings to a native Ruby type
+      # using the type indication in the line definition. It will use a custom connverter
+      # method if one is available.
       def convert_value(value, capture_definition)
-        custom_converter_method = "convert_#{capture_definition[:type]}".to_sym
-        if respond_to?(custom_converter_method) 
-          send(custom_converter_method, value, capture_definition)
-        elsif !value.nil? 
-          case capture_definition[:type]
-          when :decimal;  value.to_f
-          when :float;    value.to_f
-          when :double;   value.to_f
-          when :integer;  value.to_i
-          when :int;      value.to_i
-          when :symbol;   value.to_sym
-          else;           value.to_s
-          end
-        else 
-          nil
-        end
+        return capture_definition[:default] if value.nil?
+        custom_converter_method = :"convert_#{capture_definition[:type]}"
+        send(custom_converter_method, value, capture_definition)
       end
-      
+
+      def convert_string(value, capture_definition);  value; end
+      def convert_float(value, capture_definition);   value.to_f; end
+      def convert_decimal(value, capture_definition); value.to_f; end
+      def convert_int(value, capture_definition);     value.to_i; end
+      def convert_integer(value, capture_definition); value.to_i; end
+      def convert_sym(value, capture_definition);     value.to_sym; end
+      def convert_symbol(value, capture_definition);  value.to_sym; end
+
+      # Converts :eval field, which should evaluate to a hash.
       def convert_eval(value, capture_definition)
-        eval(value).inject({}) { |h, (k, v)| h[k.to_sym] = v; h} 
+        eval(value).inject({}) { |h, (k, v)| h[k.to_sym] = v; h}
       rescue SyntaxError
         nil
       end
-      
-      # Slow default method to parse timestamps
+
+      # Slow default method to parse timestamps.
+      # Reimplement this function in a file format specific Request class
+      # to improve the timestamp parsing speed.
       def convert_timestamp(value, capture_definition)
-        DateTime.parse(value).strftime('%Y%m%d%H%M%S').to_i unless value.nil?
+        DateTime.parse(value).strftime('%Y%m%d%H%M%S').to_i
       end
-      
+
+      # Converts traffic fields to (whole) bytes based on the given unit.
       def convert_traffic(value, capture_definition)
-        return nil if value.nil?
         case capture_definition[:unit]
+        when nil, :b, :B, :byte      then value.to_i
         when :GB, :G, :gigabyte      then (value.to_f * 1000_000_000).round
         when :GiB, :gibibyte         then (value.to_f * (2 ** 30)).round
         when :MB, :M, :megabyte      then (value.to_f * 1000_000).round
         when :MiB, :mebibyte         then (value.to_f * (2 ** 20)).round
         when :KB, :K, :kilobyte, :kB then (value.to_f * 1000).round
         when :KiB, :kibibyte         then (value.to_f * (2 ** 10)).round
-        else                              value.to_i
+        else raise "Unknown traffic unit"
         end
       end
-      
+
+      # Convert duration fields to float, and make sure the values are in seconds.
       def convert_duration(value, capture_definition)
-        return nil if value.nil?
         case capture_definition[:unit]
+        when nil, :sec, :s     then value.to_f
         when :microsec, :musec then value.to_f / 1000000.0
         when :msec, :millisec  then value.to_f / 1000.0
-        else                        value.to_f
+        else raise "Unknown duration unit"
         end
       end
     end
-  
-    include RequestLogAnalyzer::FileFormat::Awareness
+
+    # Install the default converter methods
     include Converters
-  
-    attr_reader :lines
-    attr_reader :attributes
-        
-    # Initializes a new Request object. 
+
+    attr_reader :lines, :attributes, :file_format
+
+    # Initializes a new Request object.
     # It will apply the the provided FileFormat module to this instance.
     def initialize(file_format, attributes = {})
-      @lines = []
-      @attributes = attributes
-      register_file_format(file_format)
+      @lines       = []
+      @attributes  = attributes
+      @file_format = file_format
     end
-    
+
     # Creates a new request that was parsed from the log with the given FileFormat. The hashes
     # that are passed to this function are added as lines to this request.
     def self.create(file_format, *hashes)
@@ -85,9 +87,11 @@ module RequestLogAnalyzer
       hashes.flatten.each { |hash| request << hash }
       return request
     end
-     
-    # Adds another line to the request.
-    # The line should be provides as a hash of the fields parsed from the line.
+
+    # Adds another line to the request when it is parsed in the LogParser.
+    #
+    # The line should be provided as a hash with the attributes line_definition, :captures,
+    # :lineno and :source set. This function is called from LogParser.
     def add_parsed_line (parsed_line)
       value_hash = parsed_line[:line_definition].convert_captured_values(parsed_line[:captures], self)
       value_hash[:line_type] = parsed_line[:line_definition].name
@@ -95,72 +99,75 @@ module RequestLogAnalyzer
       value_hash[:source] = parsed_line[:source]
       add_line_hash(value_hash)
     end
-    
+
+    # Adds another line to the request using a plain hash.
+    #
+    # The line should be provides as a hash of the fields parsed from the line.
     def add_line_hash(value_hash)
       @lines << value_hash
       @attributes = value_hash.merge(@attributes)
     end
-    
-    
+
+    # Adds another line to the request. This method switches automatically between
+    # the add_line_hash and add_parsed_line based on the keys of the provided hash.
     def <<(hash)
       hash[:line_definition] ? add_parsed_line(hash) : add_line_hash(hash)
     end
-    
+
     # Checks whether the given line type was parsed from the log file for this request
     def has_line_type?(line_type)
       return true if @lines.length == 1 && @lines[0][:line_type] == line_type.to_sym
-      
       @lines.detect { |l| l[:line_type] == line_type.to_sym }
     end
-    
+
     alias :=~ :has_line_type?
-    
+
     # Returns the value that was captured for the "field" of this request.
     # This function will return the first value that was captured if the field
     # was captured in multiple lines
     def first(field)
       @attributes[field]
     end
-    
+
     alias :[] :first
-    
+
     # Returns an array of all the "field" values that were captured for this request
     def every(field)
       @lines.inject([]) { |result, fields| result << fields[field] if fields.has_key?(field); result }
-    end    
-    
-    # Returns true if this request does not yet contain any parsed lines. This should only occur 
+    end
+
+    # Returns true if this request does not yet contain any parsed lines. This should only occur
     # during parsing. An empty request should never be sent to the aggregators
     def empty?
       @lines.length == 0
     end
-    
+
     # Checks whether this request is completed. A completed request contains both a parsed header
-    # line and a parsed footer line. Not that calling this function in single line mode will always 
+    # line and a parsed footer line. Not that calling this function in single line mode will always
     # return false.
     def completed?
       header_found, footer_found = false, false
-      @lines.each do |line| 
+      @lines.each do |line|
         line_def = file_format.line_definitions[line[:line_type]]
         header_found = true if line_def.header
-        footer_found = true if line_def.footer        
+        footer_found = true if line_def.footer
       end
-      header_found && footer_found      
+      header_found && footer_found
     end
-    
+
     # This function is called before a Requests is yielded.
     def validate
     end
-    
-    # Returns the first timestamp encountered in a request. 
+
+    # Returns the first timestamp encountered in a request.
     def timestamp
       first(:timestamp)
     end
-    
+
     def first_lineno
       @lines.map { |line| line[:lineno] }.reject { |v| v.nil? }.min
     end
-    
+
     def last_lineno
       @lines.map { |line| line[:lineno] }.reject { |v| v.nil? }.max
     end

data/lib/request_log_analyzer/source/database_loader.rb

@@ -2,7 +2,7 @@ require 'rubygems'
 require 'activerecord'
 
 module RequestLogAnalyzer::Source
-  
+
   # Active Resource hook
   class Request < ActiveRecord::Base
     has_many :completed_lines
@@ -26,8 +26,7 @@ module RequestLogAnalyzer::Source
   # The Database class gets log data from the database.
   class DatabaseLoader < Base
 
-    attr_reader :source_files
-    attr_reader :requests
+    attr_reader :source_files, :file_format, :requests
 
     # Initializes the log file parser instance.
     # It will apply the language specific FileFormat module to this instance. It will use the line
@@ -35,16 +34,13 @@ module RequestLogAnalyzer::Source
     #
     # <tt>format</tt>:: The current file format instance
     # <tt>options</tt>:: A hash of options that are used by the parser
-    def initialize(format, options = {})      
-      @line_definitions = {}
-      @options          = options
+    def initialize(format, options = {})
+      super(format, options)
       @source_files     = options[:source_files]
       @parsed_requests  = 0
       @requests         = []
-      
-      self.register_file_format(format)
     end
-    
+
     # Reads the input, which can either be a file, sequence of files or STDIN to parse
     # lines specified in the FileFormat. This lines will be combined into Request instances,
     # that will be yielded. The actual parsing occurs in the parse_io method.
@@ -56,10 +52,10 @@ module RequestLogAnalyzer::Source
         RequestLogAnalyzer::Source::Request.find(:all).each do |request|
           @parsed_requests += 1
           @progress_handler.call(:progress, @parsed_requests) if @progress_handler
-          
+
           yield request.convert(self.file_format)
         end
-      
+
       @progress_handler.call(:finished, @source_files) if @progress_handler
     end
 
@@ -70,15 +66,15 @@ module RequestLogAnalyzer::Source
     end
 
     # Add a block to this method to install a warning handler while parsing,
-    # <tt>proc</tt>:: The proc that will be called to handle parse warning messages    
+    # <tt>proc</tt>:: The proc that will be called to handle parse warning messages
     def warning=(proc)
       @warning_handler = proc
     end
 
     # This method is called by the parser if it encounteres any parsing problems.
-    # It will call the installed warning handler if any. 
+    # It will call the installed warning handler if any.
     #
-    # By default, RequestLogAnalyzer::Controller will install a warning handler 
+    # By default, RequestLogAnalyzer::Controller will install a warning handler
     # that will pass the warnings to each aggregator so they can do something useful
     # with it.
     #

data/lib/request_log_analyzer/source/log_parser.rb

@@ -1,13 +1,13 @@
 module RequestLogAnalyzer::Source
-  
+
   # The LogParser class reads log data from a given source and uses a file format definition
-  # to parse all relevent information about requests from the file. A FileFormat module should 
+  # to parse all relevent information about requests from the file. A FileFormat module should
   # be provided that contains the definitions of the lines that occur in the log data.
   #
-  # De order in which lines occur is used to combine lines to a single request. If these lines 
-  # are mixed, requests cannot be combined properly. This can be the case if data is written to 
-  # the log file simultaneously by different mongrel processes. This problem is detected by the 
-  # parser. It will emit warnings when this occurs. LogParser supports multiple parse strategies 
+  # De order in which lines occur is used to combine lines to a single request. If these lines
+  # are mixed, requests cannot be combined properly. This can be the case if data is written to
+  # the log file simultaneously by different mongrel processes. This problem is detected by the
+  # parser. It will emit warnings when this occurs. LogParser supports multiple parse strategies
   # that deal differently with this problem.
   class LogParser < Base
 
@@ -15,7 +15,7 @@ module RequestLogAnalyzer::Source
 
     # The default parse strategy that will be used to parse the input.
     DEFAULT_PARSE_STRATEGY = 'assume-correct'
-    
+
     # All available parse strategies.
     PARSE_STRATEGIES = ['cautious', 'assume-correct']
 
@@ -27,42 +27,44 @@ module RequestLogAnalyzer::Source
     #
     # <tt>format</tt>:: The current file format instance
     # <tt>options</tt>:: A hash of options that are used by the parser
-    def initialize(format, options = {})      
-      @line_definitions = {}
-      @options          = options
+    def initialize(format, options = {})
+      super(format, options)
       @parsed_lines     = 0
       @parsed_requests  = 0
       @skipped_lines    = 0
       @skipped_requests = 0
+      @current_request  = nil
+      @current_source   = nil
       @current_file     = nil
       @current_lineno   = nil
       @source_files     = options[:source_files]
-      
+      @progress_handler = nil
+
       @options[:parse_strategy] ||= DEFAULT_PARSE_STRATEGY
       raise "Unknown parse strategy" unless PARSE_STRATEGIES.include?(@options[:parse_strategy])
-      
-      self.register_file_format(format)
     end
-    
+
     # Reads the input, which can either be a file, sequence of files or STDIN to parse
     # lines specified in the FileFormat. This lines will be combined into Request instances,
     # that will be yielded. The actual parsing occurs in the parse_io method.
     # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
     def each_request(options = {}, &block) # :yields: :request, request
-      
+
       case @source_files
       when IO
-        puts "Parsing from the standard input. Press CTRL+C to finish." # FIXME: not here
-        parse_stream(@source_files, options, &block) 
+        if @source_files == $stdin
+          puts "Parsing from the standard input. Press CTRL+C to finish." # FIXME: not here
+        end
+        parse_stream(@source_files, options, &block)
       when String
-        parse_file(@source_files, options, &block) 
+        parse_file(@source_files, options, &block)
       when Array
-        parse_files(@source_files, options, &block) 
+        parse_files(@source_files, options, &block)
       else
         raise "Unknown source provided"
       end
     end
-    
+
     # Make sure the Enumerable methods work as expected
     alias_method :each, :each_request
 
@@ -73,12 +75,12 @@ module RequestLogAnalyzer::Source
     def parse_files(files, options = {}, &block) # :yields: request
       files.each { |file| parse_file(file, options, &block) }
     end
-    
+
     # Check if a file has a compressed extention in the filename.
     # If recognized, return the command string used to decompress the file
     def decompress_file?(filename)
       nice_command = "nice -n 5"
-      
+
       return "#{nice_command} gunzip -c -d #{filename}" if filename.match(/\.tar.gz$/) || filename.match(/\.tgz$/) || filename.match(/\.gz$/)
       return "#{nice_command} bunzip2 -c -d #{filename}" if filename.match(/\.bz2$/)
       return "#{nice_command} unzip -p #{filename}" if filename.match(/\.zip$/)
@@ -97,27 +99,32 @@ module RequestLogAnalyzer::Source
     # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
     def parse_file(file, options = {}, &block)
 
-      @progress_handler = @dormant_progress_handler
       @current_source = File.expand_path(file)
-      @progress_handler.call(:started, file)                  if @progress_handler
       @source_changes_handler.call(:started, @current_source) if @source_changes_handler
-      
+
       if decompress_file?(file).empty?
+
+        @progress_handler = @dormant_progress_handler
+        @progress_handler.call(:started, file) if @progress_handler
+
         File.open(file, 'r') { |f| parse_io(f, options, &block) }
+
+        @progress_handler.call(:finished, file) if @progress_handler
+        @progress_handler = nil
       else
         IO.popen(decompress_file?(file), 'r') { |f| parse_io(f, options, &block) }
       end
-      
+
       @source_changes_handler.call(:finished, @current_source) if @source_changes_handler
-      @progress_handler.call(:finished, file)                  if @progress_handler
+
       @current_source = nil
-      @progress_handler = nil
+
     end
 
     # Parses an IO stream. It will simply call parse_io. This function does not support progress updates
     # because the length of a stream is not known.
     # <tt>stream</tt>:: The IO stream that should be parsed.
-    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.    
+    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
     def parse_stream(stream, options = {}, &block)
       parse_io(stream, options, &block)
     end
@@ -137,16 +144,16 @@ module RequestLogAnalyzer::Source
     def parse_io(io, options = {}, &block) # :yields: request
       @current_lineno = 1
       while line = io.gets
-        @progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno % 127 == 0
-        
+        @progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno % 255 == 0
+
         if request_data = file_format.parse_line(line) { |wt, message| warn(wt, message) }
           @parsed_lines += 1
           update_current_request(request_data.merge(:source => @current_source, :lineno => @current_lineno), &block)
         end
-        
+
         @current_lineno += 1
       end
-      
+
       warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
       @current_lineno = nil
     end
@@ -162,7 +169,7 @@ module RequestLogAnalyzer::Source
     def warning=(proc)
       @warning_handler = proc
     end
-    
+
     # Add a block to this method to install a source change handler while parsing,
     # <tt>proc</tt>:: The proc that will be called to handle source changes
     def source_changes=(proc)
@@ -170,9 +177,9 @@ module RequestLogAnalyzer::Source
     end
 
     # This method is called by the parser if it encounteres any parsing problems.
-    # It will call the installed warning handler if any. 
+    # It will call the installed warning handler if any.
     #
-    # By default, RequestLogAnalyzer::Controller will install a warning handler 
+    # By default, RequestLogAnalyzer::Controller will install a warning handler
     # that will pass the warnings to each aggregator so they can do something useful
     # with it.
     #
@@ -184,24 +191,24 @@ module RequestLogAnalyzer::Source
 
     protected
 
-    # Combines the different lines of a request into a single Request object. It will start a 
-    # new request when a header line is encountered en will emit the request when a footer line 
+    # Combines the different lines of a request into a single Request object. It will start a
+    # new request when a header line is encountered en will emit the request when a footer line
     # is encountered.
     #
     # Combining the lines is done using heuristics. Problems can occur in this process. The
     # current parse strategy defines how these cases are handled.
     #
     # When using the 'assume-correct' parse strategy (default):
-    # - Every line that is parsed before a header line is ignored as it cannot be included in 
+    # - Every line that is parsed before a header line is ignored as it cannot be included in
     #   any request. It will emit a :no_current_request warning.
-    # - If a header line is found before the previous requests was closed, the previous request 
+    # - If a header line is found before the previous requests was closed, the previous request
     #   will be yielded and a new request will be started.
     #
     # When using the 'cautious' parse strategy:
-    # - Every line that is parsed before a header line is ignored as it cannot be included in 
+    # - Every line that is parsed before a header line is ignored as it cannot be included in
     #   any request. It will emit a :no_current_request warning.
     # - A header line that is parsed before a request is closed by a footer line, is a sign of
-    #   an unproperly ordered file. All data that is gathered for the request until then is 
+    #   an unproperly ordered file. All data that is gathered for the request until then is
     #   discarded and the next request is ignored as well. An :unclosed_request warning is
     #   emitted.
     #
@@ -228,7 +235,7 @@ module RequestLogAnalyzer::Source
           @current_request << request_data
           if footer_line?(request_data)
             handle_request(@current_request, &block) # yield @current_request
-            @current_request = nil 
+            @current_request = nil
           end
         else
           @skipped_lines += 1
@@ -236,8 +243,8 @@ module RequestLogAnalyzer::Source
         end
       end
     end
-    
-    # Handles the parsed request by sending it into the pipeline. 
+
+    # Handles the parsed request by sending it into the pipeline.
     #
     # - It will call RequestLogAnalyzer::Request#validate on the request instance
     # - It will send the request into the pipeline, checking whether it was accepted by all the filters.
@@ -249,19 +256,19 @@ module RequestLogAnalyzer::Source
       request.validate
       accepted = block_given? ? yield(request) : true
       @skipped_requests += 1 unless accepted
-    end    
+    end
 
     # Checks whether a given line hash is a header line according to the current file format.
-    # <tt>hash</tt>:: A hash of data that was parsed from the line.    
+    # <tt>hash</tt>:: A hash of data that was parsed from the line.
     def header_line?(hash)
       hash[:line_definition].header
     end
 
-    # Checks whether a given line hash is a footer line  according to the current file format.    
-    # <tt>hash</tt>:: A hash of data that was parsed from the line.    
+    # Checks whether a given line hash is a footer line  according to the current file format.
+    # <tt>hash</tt>:: A hash of data that was parsed from the line.
     def footer_line?(hash)
       hash[:line_definition].footer
-    end 
+    end
   end
 
 end