request-log-analyzer 1.3.4 → 1.4.0

data/lib/request_log_analyzer/output.rb

@@ -5,13 +5,49 @@ module RequestLogAnalyzer::Output
   def self.const_missing(const)
     RequestLogAnalyzer::load_default_class_file(self, const)
   end
+
+  # Loads a Output::Base subclass instance.
+  def self.load(file_format, *args)
+    
+    klass = nil
+    if file_format.kind_of?(RequestLogAnalyzer::Output::Base)
+      # this already is a file format! return itself
+      return file_format
+
+    elsif file_format.kind_of?(Class) && file_format.ancestors.include?(RequestLogAnalyzer::Output::Base)
+      # a usable class is provided. Use this format class.
+      klass = file_format
+
+    elsif file_format.kind_of?(String) && File.exist?(file_format)
+      # load a format from a ruby file
+      require file_format
+      const = RequestLogAnalyzer::to_camelcase(File.basename(file_format, '.rb'))
+      if RequestLogAnalyzer::FileFormat.const_defined?(const)
+        klass = RequestLogAnalyzer::Output.const_get(const)
+      elsif Object.const_defined?(const)
+        klass = Object.const_get(const)
+      else
+        raise "Cannot load class #{const} from #{file_format}!"
+      end
+
+    else
+      # load a provided file format
+      klass = RequestLogAnalyzer::Output.const_get(RequestLogAnalyzer::to_camelcase(file_format))
+    end
+
+    # check the returned klass to see if it can be used
+    raise "Could not load a file format from #{file_format.inspect}" if klass.nil?
+    raise "Invalid FileFormat class" unless klass.kind_of?(Class) && klass.ancestors.include?(RequestLogAnalyzer::Output::Base)
+
+    klass.create(*args) # return an instance of the class
+  end
   
   # Base Class used for generating output for reports.
   # All output should inherit fromt this class.
   class Base
-    
+
     attr_accessor :io, :options, :style
-    
+
     # Initialize a report
     # <tt>io</tt> iO Object (file, STDOUT, etc.)
     # <tt>options</tt> Specific style options
@@ -27,32 +63,37 @@ module RequestLogAnalyzer::Output
       @style = @style.merge(temp_style)
       yield(self) if block_given?
       @style = old_style
-    end    
-    
+    end
+
     # Generate a header for a report
     def header
     end
-    
-    # Generate the footer of a report 
+
+    # Generate the footer of a report
     def footer
     end
 
+    def slice_results(array)
+      return array if options[:amount] == :all
+      return array.slice(0, options[:amount]) # otherwise
+    end
+
     # Generate a report table and push it into the output object.
     # Yeilds a rows array into which the rows can be pushed
     # <tt>*colums<tt> Array of Column hashes (see Column options).
     # <tt>&block</tt>: A block yeilding the rows.
-    # 
+    #
     # === Column options
     # Columns is an array of hashes containing the column definitions.
     # * <tt>:align</tt> Alignment :left or :right
     # * <tt>:treshold</tt> Width in characters or :rest
     # * <tt>:type</tt> :ratio or nil
     # * <tt>:width</tt> Width in characters or :rest
-    # 
+    #
     # === Example
     # The output object should support table definitions:
     #
-    # output.table({:align => :left}, {:align => :right }, {:align => :right}, {:type => :ratio, :width => :rest}) do |rows|        
+    # output.table({:align => :left}, {:align => :right }, {:align => :right}, {:type => :ratio, :width => :rest}) do |rows|
     #   sorted_frequencies.each do |(cat, count)|
     #     rows << [cat, "#{count} hits", '%0.1f%%' % ((count.to_f / total_hits.to_f) * 100.0), (count.to_f / total_hits.to_f)]
     #   end
@@ -60,13 +101,13 @@ module RequestLogAnalyzer::Output
     #
     def table(*columns, &block)
     end
-    
+
     protected
     # Check if a given table defination hash includes a header (title)
     # <tt>columns</tt> The columns hash
     def table_has_header?(columns)
-      columns.any? { |column| !column[:title].nil? } 
+      columns.any? { |column| !column[:title].nil? }
     end
-    
+
   end
 end

data/lib/request_log_analyzer/request.rb

@@ -1,71 +1,85 @@
 module RequestLogAnalyzer
-  
-  # The Request class represents a parsed request from the log file. 
+
+  # The Request class represents a parsed request from the log file.
   # Instances are created by the LogParser and are passed to the different aggregators, so they
-  # can do their aggregating work. 
+  # can do their aggregating work.
   #
   # This class provides several methods to access the data that was parsed from the log files.
   # Request#first(field_name) returns the first (only) value corresponding to the given field
   # Request#every(field_name) returns all values corresponding to the given field name as array.
   class Request
-  
+
     module Converters
-      
+
+      # Default converter function, which converts the parsed strings to a native Ruby type
+      # using the type indication in the line definition. It will use a custom connverter
+      # method if one is available.
       def convert_value(value, capture_definition)
-        custom_converter_method = "convert_#{capture_definition[:type]}".to_sym
-        if respond_to?(custom_converter_method) 
-          send(custom_converter_method, value, capture_definition)
-        elsif !value.nil? 
-          case capture_definition[:type]
-          when :decimal;  value.to_f
-          when :float;    value.to_f
-          when :double;   value.to_f
-          when :integer;  value.to_i
-          when :int;      value.to_i
-          when :symbol;   value.to_sym
-          else;           value.to_s
-          end
-        else 
-          nil
-        end
+        return capture_definition[:default] if value.nil?
+        custom_converter_method = :"convert_#{capture_definition[:type]}"
+        send(custom_converter_method, value, capture_definition)
       end
-      
+
+      def convert_string(value, capture_definition);  value; end
+      def convert_float(value, capture_definition);   value.to_f; end
+      def convert_decimal(value, capture_definition); value.to_f; end
+      def convert_int(value, capture_definition);     value.to_i; end
+      def convert_integer(value, capture_definition); value.to_i; end
+      def convert_sym(value, capture_definition);     value.to_sym; end
+      def convert_symbol(value, capture_definition);  value.to_sym; end
+
+      # Converts :eval field, which should evaluate to a hash.
       def convert_eval(value, capture_definition)
-        eval(value).inject({}) { |h, (k, v)| h[k.to_sym] = v; h} 
+        eval(value).inject({}) { |h, (k, v)| h[k.to_sym] = v; h}
       rescue SyntaxError
         nil
       end
-      
-      # Slow default method to parse timestamps
+
+      # Slow default method to parse timestamps.
+      # Reimplement this function in a file format specific Request class
+      # to improve the timestamp parsing speed.
       def convert_timestamp(value, capture_definition)
-        DateTime.parse(value).strftime('%Y%m%d%H%M%S').to_i unless value.nil?
+        DateTime.parse(value).strftime('%Y%m%d%H%M%S').to_i
+      end
+
+      # Converts traffic fields to (whole) bytes based on the given unit.
+      def convert_traffic(value, capture_definition)
+        case capture_definition[:unit]
+        when nil, :b, :B, :byte      then value.to_i
+        when :GB, :G, :gigabyte      then (value.to_f * 1000_000_000).round
+        when :GiB, :gibibyte         then (value.to_f * (2 ** 30)).round
+        when :MB, :M, :megabyte      then (value.to_f * 1000_000).round
+        when :MiB, :mebibyte         then (value.to_f * (2 ** 20)).round
+        when :KB, :K, :kilobyte, :kB then (value.to_f * 1000).round
+        when :KiB, :kibibyte         then (value.to_f * (2 ** 10)).round
+        else raise "Unknown traffic unit"
+        end
       end
-      
+
+      # Convert duration fields to float, and make sure the values are in seconds.
       def convert_duration(value, capture_definition)
-        if value.nil?
-          nil
-        elsif capture_definition[:unit] == :msec
-          value.to_f / 1000.0      
-        else
-          value.to_f
+        case capture_definition[:unit]
+        when nil, :sec, :s     then value.to_f
+        when :microsec, :musec then value.to_f / 1000000.0
+        when :msec, :millisec  then value.to_f / 1000.0
+        else raise "Unknown duration unit"
         end
       end
     end
-  
-    include RequestLogAnalyzer::FileFormat::Awareness
+
+    # Install the default converter methods
     include Converters
-  
-    attr_reader :lines
-    attr_reader :attributes
-        
-    # Initializes a new Request object. 
+
+    attr_reader :lines, :attributes, :file_format
+
+    # Initializes a new Request object.
     # It will apply the the provided FileFormat module to this instance.
     def initialize(file_format, attributes = {})
-      @lines = []
-      @attributes = attributes
-      register_file_format(file_format)
+      @lines       = []
+      @attributes  = attributes
+      @file_format = file_format
     end
-    
+
     # Creates a new request that was parsed from the log with the given FileFormat. The hashes
     # that are passed to this function are added as lines to this request.
     def self.create(file_format, *hashes)
@@ -73,9 +87,11 @@ module RequestLogAnalyzer
       hashes.flatten.each { |hash| request << hash }
       return request
     end
-     
-    # Adds another line to the request.
-    # The line should be provides as a hash of the fields parsed from the line.
+
+    # Adds another line to the request when it is parsed in the LogParser.
+    #
+    # The line should be provided as a hash with the attributes line_definition, :captures,
+    # :lineno and :source set. This function is called from LogParser.
     def add_parsed_line (parsed_line)
       value_hash = parsed_line[:line_definition].convert_captured_values(parsed_line[:captures], self)
       value_hash[:line_type] = parsed_line[:line_definition].name
@@ -83,72 +99,75 @@ module RequestLogAnalyzer
       value_hash[:source] = parsed_line[:source]
       add_line_hash(value_hash)
     end
-    
+
+    # Adds another line to the request using a plain hash.
+    #
+    # The line should be provides as a hash of the fields parsed from the line.
     def add_line_hash(value_hash)
       @lines << value_hash
       @attributes = value_hash.merge(@attributes)
     end
-    
-    
+
+    # Adds another line to the request. This method switches automatically between
+    # the add_line_hash and add_parsed_line based on the keys of the provided hash.
     def <<(hash)
       hash[:line_definition] ? add_parsed_line(hash) : add_line_hash(hash)
     end
-    
+
     # Checks whether the given line type was parsed from the log file for this request
     def has_line_type?(line_type)
       return true if @lines.length == 1 && @lines[0][:line_type] == line_type.to_sym
-      
       @lines.detect { |l| l[:line_type] == line_type.to_sym }
     end
-    
+
     alias :=~ :has_line_type?
-    
+
     # Returns the value that was captured for the "field" of this request.
     # This function will return the first value that was captured if the field
     # was captured in multiple lines
     def first(field)
       @attributes[field]
     end
-    
+
     alias :[] :first
-    
+
     # Returns an array of all the "field" values that were captured for this request
     def every(field)
       @lines.inject([]) { |result, fields| result << fields[field] if fields.has_key?(field); result }
-    end    
-    
-    # Returns true if this request does not yet contain any parsed lines. This should only occur 
+    end
+
+    # Returns true if this request does not yet contain any parsed lines. This should only occur
     # during parsing. An empty request should never be sent to the aggregators
     def empty?
       @lines.length == 0
     end
-    
+
     # Checks whether this request is completed. A completed request contains both a parsed header
-    # line and a parsed footer line. Not that calling this function in single line mode will always 
+    # line and a parsed footer line. Not that calling this function in single line mode will always
     # return false.
     def completed?
       header_found, footer_found = false, false
-      @lines.each do |line| 
+      @lines.each do |line|
         line_def = file_format.line_definitions[line[:line_type]]
         header_found = true if line_def.header
-        footer_found = true if line_def.footer        
+        footer_found = true if line_def.footer
       end
-      header_found && footer_found      
+      header_found && footer_found
     end
-    
+
     # This function is called before a Requests is yielded.
     def validate
     end
-    
-    # Returns the first timestamp encountered in a request. 
+
+    # Returns the first timestamp encountered in a request.
     def timestamp
       first(:timestamp)
     end
-    
+
     def first_lineno
       @lines.map { |line| line[:lineno] }.reject { |v| v.nil? }.min
     end
-    
+
     def last_lineno
       @lines.map { |line| line[:lineno] }.reject { |v| v.nil? }.max
     end

data/lib/request_log_analyzer/source/database_loader.rb

@@ -2,7 +2,7 @@ require 'rubygems'
 require 'activerecord'
 
 module RequestLogAnalyzer::Source
-  
+
   # Active Resource hook
   class Request < ActiveRecord::Base
     has_many :completed_lines
@@ -26,8 +26,7 @@ module RequestLogAnalyzer::Source
   # The Database class gets log data from the database.
   class DatabaseLoader < Base
 
-    attr_reader :source_files
-    attr_reader :requests
+    attr_reader :source_files, :file_format, :requests
 
     # Initializes the log file parser instance.
     # It will apply the language specific FileFormat module to this instance. It will use the line
@@ -35,16 +34,13 @@ module RequestLogAnalyzer::Source
     #
     # <tt>format</tt>:: The current file format instance
     # <tt>options</tt>:: A hash of options that are used by the parser
-    def initialize(format, options = {})      
-      @line_definitions = {}
-      @options          = options
+    def initialize(format, options = {})
+      super(format, options)
       @source_files     = options[:source_files]
       @parsed_requests  = 0
       @requests         = []
-      
-      self.register_file_format(format)
     end
-    
+
     # Reads the input, which can either be a file, sequence of files or STDIN to parse
     # lines specified in the FileFormat. This lines will be combined into Request instances,
     # that will be yielded. The actual parsing occurs in the parse_io method.
@@ -56,10 +52,10 @@ module RequestLogAnalyzer::Source
         RequestLogAnalyzer::Source::Request.find(:all).each do |request|
           @parsed_requests += 1
           @progress_handler.call(:progress, @parsed_requests) if @progress_handler
-          
+
           yield request.convert(self.file_format)
         end
-      
+
       @progress_handler.call(:finished, @source_files) if @progress_handler
     end
 
@@ -70,15 +66,15 @@ module RequestLogAnalyzer::Source
     end
 
     # Add a block to this method to install a warning handler while parsing,
-    # <tt>proc</tt>:: The proc that will be called to handle parse warning messages    
+    # <tt>proc</tt>:: The proc that will be called to handle parse warning messages
     def warning=(proc)
       @warning_handler = proc
     end
 
     # This method is called by the parser if it encounteres any parsing problems.
-    # It will call the installed warning handler if any. 
+    # It will call the installed warning handler if any.
     #
-    # By default, RequestLogAnalyzer::Controller will install a warning handler 
+    # By default, RequestLogAnalyzer::Controller will install a warning handler
     # that will pass the warnings to each aggregator so they can do something useful
     # with it.
     #