request-log-analyzer 1.3.5 → 1.4.0

data/lib/request_log_analyzer/source.rb

@@ -1,7 +1,7 @@
 # The RequestLogAnalyzer::Source module contains all functionality that loads requests from a given source
 # and feed them to the pipeline for further processing. The requests (see RequestLogAnalyzer::Request) that
-# will be parsed from a source, will be piped throug filters (see RequestLogAnalyzer::Filter) and are then 
-# fed to an aggregator (see RequestLogAnalyzer::Aggregator). The source instance is thus the beginning of 
+# will be parsed from a source, will be piped throug filters (see RequestLogAnalyzer::Filter) and are then
+# fed to an aggregator (see RequestLogAnalyzer::Aggregator). The source instance is thus the beginning of
 # the RequestLogAnalyzer chain.
 #
 # - The base class for all sources is RequestLogAnalyzer::Source::Base. All source classes should inherit from this class.
@@ -14,15 +14,12 @@ module RequestLogAnalyzer::Source
   def self.const_missing(const)
     RequestLogAnalyzer::load_default_class_file(self, const)
   end
-  
+
   # The base Source class. All other sources should inherit from this class.
   #
   # A source implememtation should at least implement the each_request method, which should yield
   # RequestLogAnalyzer::Request instances that will be fed through the pipleine.
   class Base
-    
-    # Make the Spurce instance aware of the current file format
-    include RequestLogAnalyzer::FileFormat::Awareness
 
     # A hash of options
     attr_reader :options
@@ -42,23 +39,24 @@ module RequestLogAnalyzer::Source
     # The total number of skipped requests because of filters.
     attr_reader :skipped_requests
 
-
+    # The FileFormat instance that describes the format of this source.
+    attr_reader :file_format
 
     # Initializer, which will register the file format and save any options given as a hash.
     # <tt>format</tt>:: The file format instance
     # <tt>options</tt>:: A hash of options that can be used by a specific Source implementation
     def initialize(format, options = {})
-      @options    = options
-      register_file_format(format)
+      @options     = options
+      @file_format = format
     end
-    
+
     # The prepare method is called before the RequestLogAnalyzer::Source::Base#each_request method is called.
     # Use this method to implement any initialization that should occur before this source can produce Request
     # instances.
     def prepare
     end
-    
-    # This function is called to actually produce the requests that will be send into the pipeline. 
+
+    # This function is called to actually produce the requests that will be send into the pipeline.
     # The implementation should yield instances of RequestLogAnalyzer::Request.
     # <tt>options</tt>:: A Hash of options that can be used in the implementation.
     def each_request(options = {}, &block) # :yields: request

data/lib/request_log_analyzer/tracker/duration.rb

@@ -3,12 +3,11 @@ module RequestLogAnalyzer::Tracker
   # Analyze the duration of a specific attribute
   #
   # === Options
-  # * <tt>:amount</tt> The amount of lines in the report
   # * <tt>:category</tt> Proc that handles request categorization for given fileformat (REQUEST_CATEGORIZER)
   # * <tt>:duration</tt> The field containing the duration in the request hash.
   # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
   # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
-  # * <tt>:title</tt> Title do be displayed above the report  
+  # * <tt>:title</tt> Title do be displayed above the report
   # * <tt>:unless</tt> Handle request if this proc is false for the handled request.
   #
   # The items in the update request hash are set during the creation of the Duration tracker.
@@ -21,141 +20,62 @@ module RequestLogAnalyzer::Tracker
   #  EmployeeController#index.html [GET]            |    5802 |  1477.32s |     0.25s
   #  .............
   class Duration < Base
-    
+
+    include RequestLogAnalyzer::Tracker::StatisticsTracking
+
     attr_reader :categories
 
     # Check if duration and catagory option have been received,
     def prepare
       raise "No duration field set up for category tracker #{self.inspect}" unless options[:duration]
-      raise "No categorizer set up for duration tracker #{self.inspect}" unless options[:category]
+      raise "No categorizer set up for duration tracker #{self.inspect}"    unless options[:category]
 
-      @categorizer  = options[:category].respond_to?(:call) ? options[:category] : lambda { |request| request[options[:category]] }
-      @durationizer = options[:duration].respond_to?(:call) ? options[:duration] : lambda { |request| request[options[:duration]] }
-      @categories = {}
+      unless options[:multiple]
+        @categorizer  = create_lambda(options[:category])
+        @durationizer = create_lambda(options[:duration])
+      end
+      
+      @categories   = {}
     end
 
     # Get the duration information fron the request and store it in the different categories.
     # <tt>request</tt> The request.
     def update(request)
       if options[:multiple]
-        categories = request.every(options[:category])
-        durations  = request.every(options[:duration])
-        
-        if categories.length == durations.length
-          categories.each_with_index do |category, index|
-            @categories[category] ||= {:hits => 0, :cumulative => 0.0}
-            @categories[category][:hits] += 1
-            @categories[category][:cumulative] += durations[index]
-          end
-        else
-          raise "Capture mismatch for multiple values in a request"
-        end
+        found_categories = request.every(options[:category])
+        found_durations  = request.every(options[:duration])
+        raise "Capture mismatch for multiple values in a request" unless found_categories.length == found_durations.length
+        found_categories.each_with_index { |cat, index| update_statistics(cat, found_durations[index]) }
       else
         category = @categorizer.call(request)
         duration = @durationizer.call(request)
-  
-        if duration.kind_of?(Numeric) && !category.nil?
-          @categories[category] ||= {:hits => 0, :cumulative => 0.0, :min => duration, :max => duration }
-          @categories[category][:hits] += 1
-          @categories[category][:cumulative] += duration
-          @categories[category][:min] = duration if duration < @categories[category][:min]
-          @categories[category][:max] = duration if duration > @categories[category][:max]
-        end
+        update_statistics(category, duration) if duration.kind_of?(Numeric) && category
       end
     end
-    
-    # Get the number of hits of a specific category.
-    # <tt>cat</tt> The category
-    def hits(cat)
-      categories[cat][:hits]
-    end
-    
-    # Get the total duration of a specific category.
-    # <tt>cat</tt> The category
-    def cumulative_duration(cat)
-      categories[cat][:cumulative]
-    end
-
-    # Get the minimal duration of a specific category.
-    # <tt>cat</tt> The category
-    def min_duration(cat)
-      categories[cat][:min]
-    end
-
-    # Get the maximum duration of a specific category.
-    # <tt>cat</tt> The category
-    def max_duration(cat)
-      categories[cat][:max]
-    end
-
-    # Get the average duration of a specific category.
-    # <tt>cat</tt> The category
-    def average_duration(cat)
-      categories[cat][:cumulative] / categories[cat][:hits]  
-    end
-    
-    # Get the average duration of a all categories.
-    def overall_average_duration
-      overall_cumulative_duration / overall_hits
-    end
-    
-    # Get the cumlative duration of a all categories.
-    def overall_cumulative_duration
-      categories.inject(0.0) { |sum, (name, cat)| sum + cat[:cumulative] }  
-    end
-    
-    # Get the total hits of a all categories.
-    def overall_hits
-      categories.inject(0) { |sum, (name, cat)| sum + cat[:hits] }
-    end
 
-    # Return categories sorted by hits.
-    def sorted_by_hits
-      sorted_by(:hits)
-    end
-
-    # Return categories sorted by cumulative duration.
-    def sorted_by_cumulative
-      sorted_by(:cumulative)
-    end
-
-    # Return categories sorted by cumulative duration.
-    def sorted_by_average
-      sorted_by { |cat| cat[:cumulative] / cat[:hits] }
-    end
-          
-    # Return categories sorted by a given key.
-    # <tt>by</tt> The key.
-    def sorted_by(by = nil) 
-      if block_given?
-        categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) } 
-      else
-        categories.sort { |a, b| b[1][by] <=> a[1][by] } 
-      end
-    end
-    
     # Block function to build a result table using a provided sorting function.
     # <tt>output</tt> The output object.
     # <tt>amount</tt> The number of rows in the report table (default 10).
     # === Options
     #  * </tt>:title</tt> The title of the table
     #  * </tt>:sort</tt> The key to sort on (:hits, :cumulative, :average, :min or :max)
-    def report_table(output, amount = 10, options = {}, &block)
-    
-      output.title(options[:title])
-    
-      top_categories = @categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }.slice(0...amount)
-      output.table({:title => 'Category', :width => :rest}, 
-            {:title => 'Hits',       :align => :right, :highlight => (options[:sort] == :hits), :min_width => 4}, 
-            {:title => 'Cumulative', :align => :right, :highlight => (options[:sort] == :cumulative), :min_width => 10}, 
-            {:title => 'Average',    :align => :right, :highlight => (options[:sort] == :average), :min_width => 8},
-            {:title => 'Min',        :align => :right, :highlight => (options[:sort] == :min)}, 
-            {:title => 'Max',        :align => :right, :highlight => (options[:sort] == :max)}) do |rows|
-      
-        top_categories.each do |(cat, info)|
-          rows << [cat, info[:hits], "%0.02fs" % info[:cumulative], "%0.02fs" % (info[:cumulative] / info[:hits]),
-                    "%0.02fs" % info[:min], "%0.02fs" % info[:max]]
-        end        
+    def report_table(output, sort, options = {}, &block)
+      output.puts
+      top_categories = output.slice_results(sorted_by(sort))
+      output.with_style(:top_line => true) do
+        output.table(*statistics_header(:title => options[:title], :highlight => sort)) do |rows|
+          top_categories.each { |(cat, info)| rows << statistics_row(cat) }
+        end
+      end
+    end
+
+    # Display a duration
+    def display_value(time)
+      case time
+      when nil       then '-'
+      when 0...60    then "%0.02fs" % time
+      when 60...3600 then "%dm%02ds" % [time / 60, (time % 60).round]
+      else                "%dh%02dm%02ds" % [time / 3600, (time % 3600) / 60, (time % 60).round]
       end
     end
 
@@ -164,33 +84,20 @@ module RequestLogAnalyzer::Tracker
     # Any options for the report should have been set during initialize.
     # <tt>output</tt> The output object
     def report(output)
-
-      options[:title]  ||= 'Request duration'
-      options[:report] ||= [:cumulative, :average]
-      options[:top]    ||= 20
-  
-      options[:report].each do |report|
-        case report
-        when :average
-          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by average time", :sort => :average) { |cat| cat[:cumulative] / cat[:hits] }  
-        when :cumulative
-          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by cumulative time", :sort => :cumulative) { |cat| cat[:cumulative] }
-        when :hits
-          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by hits", :sort => :hits) { |cat| cat[:hits] }
-        else
-          raise "Unknown duration report specified: #{report}!"
-        end
+      sortings = output.options[:sort] || [:sum, :mean]
+      sortings.each do |sorting|
+        report_table(output, sorting, :title => "#{title} - by #{sorting}")
       end
     end
-    
+
     # Returns the title of this tracker for reports
     def title
       options[:title]  || 'Request duration'
     end
-    
+
     # Returns all the categories and the tracked duration as a hash than can be exported to YAML
     def to_yaml_object
-      return nil if @categories.empty?      
+      return nil if @categories.empty?
       @categories
     end
   end

data/lib/request_log_analyzer/tracker/frequency.rb

@@ -1,10 +1,9 @@
 module RequestLogAnalyzer::Tracker
-  
+
   # Catagorize requests by frequency.
-  # Count and analyze requests for a specific attribute 
+  # Count and analyze requests for a specific attribute
   #
   # === Options
-  # * <tt>:amount</tt> The amount of lines in the report
   # * <tt>:category</tt> Proc that handles the request categorization.
   # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
   # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
@@ -23,41 +22,42 @@ module RequestLogAnalyzer::Tracker
   #  DELETE |    512 hits (1.1%)  |
   class Frequency < Base
 
-    attr_reader :frequencies
+    attr_reader :categories
 
     # Check if categories are set up
     def prepare
       raise "No categorizer set up for category tracker #{self.inspect}" unless options[:category]
-      @frequencies = {}
-      if options[:all_categories].kind_of?(Enumerable)
-        options[:all_categories].each { |cat| @frequencies[cat] = 0 }
-      end
+      @categorizer = create_lambda(options[:category])
+      
+      # Initialize the categories. Use the list of category names to 
+      @categories = {}
+      options[:all_categories].each { |cat| @categories[cat] = 0 } if options[:all_categories].kind_of?(Enumerable)
     end
-            
-    # Check HTTP method of a request and store that in the frequencies hash.
+
+    # Check HTTP method of a request and store that in the categories hash.
     # <tt>request</tt> The request.
     def update(request)
-      cat = options[:category].respond_to?(:call) ? options[:category].call(request) : request[options[:category]]
-      if !cat.nil? || options[:nils]
-        @frequencies[cat] ||= 0
-        @frequencies[cat] += 1
+      cat = @categorizer.call(request)
+      if cat || options[:nils]
+        @categories[cat] ||= 0
+        @categories[cat] += 1
       end
     end
 
     # Return the amount of times a HTTP method has been encountered
     # <tt>cat</tt> The HTTP method (:get, :put, :post or :delete)
     def frequency(cat)
-      frequencies[cat] || 0
+      categories[cat] || 0
     end
-    
+
     # Return the overall frequency
     def overall_frequency
-      frequencies.inject(0) { |carry, item| carry + item[1] }
+      categories.inject(0) { |carry, item| carry + item[1] }
     end
-    
+
     # Return the methods sorted by frequency
     def sorted_by_frequency
-      @frequencies.sort { |a, b| b[1] <=> a[1] }
+      @categories.sort { |a, b| b[1] <=> a[1] }
     end
 
     # Generate a HTTP method frequency report to the given output object.
@@ -65,29 +65,28 @@ module RequestLogAnalyzer::Tracker
     # <tt>output</tt> The output object
     def report(output)
       output.title(options[:title]) if options[:title]
-    
-      if @frequencies.empty?
-        output << "None found.\n" 
+
+      if @categories.empty?
+        output << "None found.\n"
       else
-        sorted_frequencies = @frequencies.sort { |a, b| b[1] <=> a[1] }
-        total_hits         = sorted_frequencies.inject(0) { |carry, item| carry + item[1] }
-        sorted_frequencies = sorted_frequencies.slice(0...options[:amount]) if options[:amount]
+        sorted_categories = output.slice_results(sorted_by_frequency)
+        total_hits        = overall_frequency
 
-        output.table({:align => :left}, {:align => :right }, {:align => :right}, {:type => :ratio, :width => :rest}) do |rows|        
-          sorted_frequencies.each do |(cat, count)|
+        output.table({:align => :left}, {:align => :right }, {:align => :right}, {:type => :ratio, :width => :rest}) do |rows|
+          sorted_categories.each do |(cat, count)|
             rows << [cat, "#{count} hits", '%0.1f%%' % ((count.to_f / total_hits.to_f) * 100.0), (count.to_f / total_hits.to_f)]
           end
         end
 
       end
     end
-    
-    # Returns a hash with the frequencies of every category that can be exported to YAML
+
+    # Returns a hash with the categories of every category that can be exported to YAML
     def to_yaml_object
-      return nil if @frequencies.empty?
-      @frequencies
+      return nil if @categories.empty?
+      @categories
     end
-    
+
     # Returns the title of this tracker for reports
     def title
       options[:title] || 'Request frequency'

data/lib/request_log_analyzer/tracker/hourly_spread.rb

@@ -1,5 +1,5 @@
 module RequestLogAnalyzer::Tracker
-  
+
   # Determines the average hourly spread of the parsed requests.
   # This spread is shown in a graph form.
   #
@@ -30,51 +30,51 @@ module RequestLogAnalyzer::Tracker
   #   ................
   class HourlySpread < Base
 
-    attr_reader :first, :last, :request_time_graph
+    attr_reader :hour_frequencies, :first, :last
 
     # Check if timestamp field is set in the options and prepare the result time graph.
     def prepare
       options[:field] ||= :timestamp
-      @request_time_graph = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+      @hour_frequencies = (0...24).map { 0 }
+      @first, @last = 99999999999999, 0
     end
 
     # Check if the timestamp in the request and store it.
     # <tt>request</tt> The request.
     def update(request)
-      request = request.attributes
-      timestamp = request[options[:field]]
-
-      @request_time_graph[timestamp.to_s[8..9].to_i] +=1
-      @first = timestamp if @first.nil? || timestamp < @first
-      @last  = timestamp if @last.nil?  || timestamp > @last
+      timestamp = request.first(options[:field])
+      @hour_frequencies[timestamp.to_s[8..9].to_i] +=1
+      @first = timestamp if timestamp < @first
+      @last  = timestamp if timestamp > @last
     end
 
     # Total amount of requests tracked
     def total_requests
-      @request_time_graph.inject(0) { |sum, value| sum + value }
+      @hour_frequencies.inject(0) { |sum, value| sum + value }
     end
-    
+
+
     # First timestamp encountered
     def first_timestamp
       DateTime.parse(@first.to_s, '%Y%m%d%H%M%S') rescue nil
     end
-    
+
     # Last timestamp encountered
     def last_timestamp
       DateTime.parse(@last.to_s, '%Y%m%d%H%M%S') rescue nil
     end
-    
+
     # Difference between last and first timestamp.
     def timespan
       last_timestamp - first_timestamp
     end
-
+    
     # Generate an hourly spread report to the given output object.
     # Any options for the report should have been set during initialize.
     # <tt>output</tt> The output object
     def report(output)
       output.title(title)
-    
+
       if total_requests == 0
         output << "None found.\n"
         return
@@ -82,23 +82,23 @@ module RequestLogAnalyzer::Tracker
 
       days = [1, timespan].max
       output.table({}, {:align => :right}, {:type => :ratio, :width => :rest, :treshold => 0.15}) do |rows|
-        @request_time_graph.each_with_index do |requests, index|
-          ratio = requests.to_f / total_requests.to_f
+        @hour_frequencies.each_with_index do |requests, index|
+          ratio            = requests.to_f / total_requests.to_f
           requests_per_day = (requests / days).ceil
-          rows << ["#{index.to_s.rjust(3)}:00", "%d hits" % requests_per_day, ratio]
+          rows << ["#{index.to_s.rjust(3)}:00", "%d hits/day" % requests_per_day, ratio]
         end
       end
     end
-    
+
     # Returns the title of this tracker for reports
     def title
       options[:title] || "Request distribution per hour"
     end
-    
+
     # Returns the found frequencies per hour as a hash for YAML exporting
     def to_yaml_object
       yaml_object = {}
-      @request_time_graph.each_with_index do |freq, hour|
+      @hour_frequencies.each_with_index do |freq, hour|
         yaml_object["#{hour}:00 - #{hour+1}:00"] = freq
       end
       yaml_object

data/lib/request_log_analyzer/tracker/timespan.rb

@@ -1,5 +1,5 @@
 module RequestLogAnalyzer::Tracker
-  
+
   # Determines the datetime of the first request and the last request
   # Also determines the amount of days inbetween these.
   #
@@ -19,32 +19,32 @@ module RequestLogAnalyzer::Tracker
   #  Total time analyzed:  7 days
   class Timespan < Base
 
-    attr_reader :first, :last, :request_time_graph
+    attr_reader :first, :last
 
     # Check if timestamp field is set in the options.
     def prepare
       options[:field] ||= :timestamp
+      @first, @last = 99999999999999, 0
     end
-            
+
     # Check if the timestamp in the request and store it.
     # <tt>request</tt> The request.
     def update(request)
       timestamp = request[options[:field]]
-
-      @first = timestamp if @first.nil? || timestamp < @first
-      @last  = timestamp if @last.nil?  || timestamp > @last
+      @first = timestamp if timestamp < @first
+      @last  = timestamp if timestamp > @last
     end
 
     # First timestamp encountered
     def first_timestamp
       DateTime.parse(@first.to_s, '%Y%m%d%H%M%S') rescue nil
     end
-    
+
     # Last timestamp encountered
     def last_timestamp
-      DateTime.parse(@last.to_s, '%Y%m%d%H%M%S') rescue nil      
+      DateTime.parse(@last.to_s, '%Y%m%d%H%M%S') rescue nil
     end
-    
+
     # Difference between last and first timestamp.
     def timespan
       last_timestamp - first_timestamp
@@ -55,27 +55,27 @@ module RequestLogAnalyzer::Tracker
     # <tt>output</tt> The output object
     def report(output)
       output.title(options[:title]) if options[:title]
-    
-      if @last && @first
-        output.with_style(:cell_separator => false) do         
+
+      if @last > 0 && @first < 99999999999999
+        output.with_style(:cell_separator => false) do
           output.table({:width => 20}, {}) do |rows|
             rows << ['First request:', first_timestamp.strftime('%Y-%m-%d %H:%M:%I')]
-            rows << ['Last request:', last_timestamp.strftime('%Y-%m-%d %H:%M:%I')]
-            rows << ['Total time analyzed:', "#{timespan.ceil} days"]                    
+            rows << ['Last request:',  last_timestamp.strftime('%Y-%m-%d %H:%M:%I')]
+            rows << ['Total time analyzed:', "#{timespan.ceil} days"]
           end
         end
       end
     end
-    
+
     # Returns the title of this tracker for reports
     def title
       options[:title] || 'Request timespan'
     end
-    
+
     # A hash that can be exported to YAML with the first and last timestamp encountered.
     def to_yaml_object
       { :first => first_timestamp, :last  =>last_timestamp }
     end
-    
+
   end
 end