request-log-analyzer 1.0.2 → 1.6.3

data/lib/request_log_analyzer/tracker/duration.rb

@@ -2,80 +2,44 @@ module RequestLogAnalyzer::Tracker
 
   # Analyze the duration of a specific attribute
   #
-  # Accepts the following options:
-  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
-  # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
-  # * <tt>:title</tt> Title do be displayed above the report  
+  # === Options
   # * <tt>:category</tt> Proc that handles request categorization for given fileformat (REQUEST_CATEGORIZER)
   # * <tt>:duration</tt> The field containing the duration in the request hash.
-  # * <tt>:amount</tt> The amount of lines in the report
+  # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+  # * <tt>:title</tt> Title do be displayed above the report
+  # * <tt>:unless</tt> Handle request if this proc is false for the handled request.
   #
   # The items in the update request hash are set during the creation of the Duration tracker.
   #
   # Example output:
-  #  Request duration - top 20 by cumulative time   ┃    Hits ┃      Sum. |      Avg.
-  #   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-  #  EmployeeController#show.html [GET]             ┃    4742 ┃  4922.56s ┃     1.04s
-  #  EmployeeController#update.html [POST]          ┃    4647 ┃  2731.23s ┃     0.59s
-  #  EmployeeController#index.html [GET]            ┃    5802 ┃  1477.32s ┃     0.25s
+  #  Request duration - top 20 by cumulative time   |    Hits |      Sum. |      Avg.
+  #  ---------------------------------------------------------------------------------
+  #  EmployeeController#show.html [GET]             |    4742 |  4922.56s |     1.04s
+  #  EmployeeController#update.html [POST]          |    4647 |  2731.23s |     0.59s
+  #  EmployeeController#index.html [GET]            |    5802 |  1477.32s |     0.25s
   #  .............
-  class Duration < RequestLogAnalyzer::Tracker::Base
-    attr_reader :categories
-  
+  class Duration < NumericValue
+
+    # Check if duration and catagory option have been received,
     def prepare
-      raise "No duration field set up for category tracker #{self.inspect}" unless options[:duration]
-      raise "No categorizer set up for duration tracker #{self.inspect}" unless options[:category]
-    
-      @categories = {}
+      options[:value] = options[:duration] if options[:duration]
+      super
     end
-  
-    def update(request)
-      category = options[:category].respond_to?(:call) ? options[:category].call(request) : request[options[:category]]
-      duration = options[:duration].respond_to?(:call) ? options[:duration].call(request) : request[options[:duration]]
-    
-      if !duration.nil? && !category.nil?
-        @categories[category] ||= {:count => 0, :total_duration => 0.0}
-        @categories[category][:count] += 1
-        @categories[category][:total_duration] += duration
-      end
-    end
-  
-    def report_table(output = STDOUT, amount = 10, options = {}, &block)
-      
-      top_categories = @categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }.slice(0...amount)
-      max_cat_length = top_categories.map { |a| a[0].length }.max || 0
-      space_left = [options[:report_width] - 33, [max_cat_length + 1, options[:title].length].max].min
-      
-      output << "\n"
-      output << "%-#{space_left+1}s┃    Hits ┃      Sum. |      Avg." % [options[:title][0...space_left]] + "\n"
-      output << green('━' * options[:report_width], options[:color]) + "\n"
-          
-      top_categories.each do |(cat, info)|
-        hits  = info[:count]
-        total = "%0.02f" % info[:total_duration]
-        avg   = "%0.02f" % (info[:total_duration] / info[:count])
-        output << "%-#{space_left+1}s┃%8d ┃%9ss ┃%9ss" % [cat[0...space_left], hits, total, avg] + "\n"
+
+    # Display a duration
+    def display_value(time)
+      case time
+      when nil       then '-'
+      when 0...60    then "%0.02fs" % time
+      when 60...3600 then "%dm%02ds" % [time / 60, (time % 60).round]
+      else                "%dh%02dm%02ds" % [time / 3600, (time % 3600) / 60, (time % 60).round]
       end
     end
-  
-    def report(output = STDOUT, report_width = 80, color = false)
 
-      options[:title]  ||= 'Request duration'
-      options[:report] ||= [:total, :average]
-      options[:top]    ||= 20
-    
-      options[:report].each do |report|
-        case report
-        when :average
-          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by average time", :color => color, :report_width => report_width) { |request| request[:total_duration] / request[:count] }  
-        when :total
-          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by cumulative time", :color => color, :report_width => report_width) { |request| request[:total_duration] }
-        when :hits
-          report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by hits", :color => color, :report_width => report_width) { |request| request[:count] }
-        else
-          output << "Unknown duration report specified\n"
-        end
-      end
-    end      
+    # Returns the title of this tracker for reports
+    def title
+      options[:title]  || 'Request duration'
+    end
   end
-end
+end

data/lib/request_log_analyzer/tracker/frequency.rb

@@ -0,0 +1,108 @@
+module RequestLogAnalyzer::Tracker
+
+  # Catagorize requests by frequency.
+  # Count and analyze requests for a specific attribute
+  #
+  # === Options
+  # * <tt>:category</tt> Proc that handles the request categorization.
+  # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+  # * <tt>:nils</tt> Track undetermined methods.
+  # * <tt>:title</tt> Title do be displayed above the report.
+  # * <tt>:unless</tt> Proc that has to return nil for a request to be passed to the tracker.
+  #
+  # The items in the update request hash are set during the creation of the Duration tracker.
+  #
+  # Example output:
+  #  HTTP methods
+  #  ----------------------------------------------------------------------
+  #  GET    |  22248 hits (46.2%) |=================
+  #  PUT    |  13685 hits (28.4%) |===========
+  #  POST   |  11662 hits (24.2%) |=========
+  #  DELETE |    512 hits (1.1%)  |
+  class Frequency < Base
+
+    attr_reader :categories
+
+    # Check if categories are set up
+    def prepare
+      options[:category] = options[:value] if options[:value] && !options[:category]
+      raise "No categorizer set up for category tracker #{self.inspect}" unless options[:category]
+      
+      @categorizer = create_lambda(options[:category]) unless options[:multiple]
+      
+      # Initialize the categories. Use the list of category names to 
+      @categories = {}
+      options[:all_categories].each { |cat| @categories[cat] = 0 } if options[:all_categories].kind_of?(Enumerable)
+    end
+
+    # Check HTTP method of a request and store that in the categories hash.
+    # <tt>request</tt> The request.
+    def update(request)
+      if options[:multiple]
+        cats = request.every(options[:category])
+        cats.each do |cat|
+          if cat || options[:nils]
+            @categories[cat] ||= 0
+            @categories[cat] += 1
+          end          
+        end
+        
+      else
+        cat = @categorizer.call(request)
+        if cat || options[:nils]
+          @categories[cat] ||= 0
+          @categories[cat] += 1
+        end
+      end
+    end
+
+    # Return the amount of times a HTTP method has been encountered
+    # <tt>cat</tt> The HTTP method (:get, :put, :post or :delete)
+    def frequency(cat)
+      categories[cat] || 0
+    end
+
+    # Return the overall frequency
+    def overall_frequency
+      categories.inject(0) { |carry, item| carry + item[1] }
+    end
+
+    # Return the methods sorted by frequency
+    def sorted_by_frequency
+      @categories.sort { |a, b| b[1] <=> a[1] }
+    end
+
+    # Generate a HTTP method frequency report to the given output object.
+    # Any options for the report should have been set during initialize.
+    # <tt>output</tt> The output object
+    def report(output)
+      output.title(options[:title]) if options[:title]
+
+      if @categories.empty?
+        output << "None found.\n"
+      else
+        sorted_categories = output.slice_results(sorted_by_frequency)
+        total_hits        = overall_frequency
+
+        output.table({:align => :left}, {:align => :right }, {:align => :right}, {:type => :ratio, :width => :rest}) do |rows|
+          sorted_categories.each do |(cat, count)|
+            rows << [cat, "#{count} hits", '%0.1f%%' % ((count.to_f / total_hits.to_f) * 100.0), (count.to_f / total_hits.to_f)]
+          end
+        end
+
+      end
+    end
+
+    # Returns a hash with the categories of every category that can be exported to YAML
+    def to_yaml_object
+      return nil if @categories.empty?
+      @categories
+    end
+
+    # Returns the title of this tracker for reports
+    def title
+      options[:title] || 'Request frequency'
+    end
+  end
+end

data/lib/request_log_analyzer/tracker/hourly_spread.rb

@@ -4,77 +4,104 @@ module RequestLogAnalyzer::Tracker
   # This spread is shown in a graph form.
   #
   # Accepts the following options:
-  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
   # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+  # * <tt>:output</tt> Direct output here (defaults to STDOUT)
+  # * <tt>:unless</tt> Proc that has to return nil for a request to be passed to the tracker.
   #
   # Expects the following items in the update request hash
   # * <tt>:timestamp</tt> in YYYYMMDDHHMMSS format.
   #
   # Example output:
   #  Requests graph - average per day per hour
-  #   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-  #    7:00 - 330 hits        : ░░░░░░░
-  #    8:00 - 704 hits        : ░░░░░░░░░░░░░░░░░
-  #    9:00 - 830 hits        : ░░░░░░░░░░░░░░░░░░░░
-  #   10:00 - 822 hits        : ░░░░░░░░░░░░░░░░░░░
-  #   11:00 - 823 hits        : ░░░░░░░░░░░░░░░░░░░
-  #   12:00 - 729 hits        : ░░░░░░░░░░░░░░░░░
-  #   13:00 - 614 hits        : ░░░░░░░░░░░░░░
-  #   14:00 - 690 hits        : ░░░░░░░░░░░░░░░░
-  #   15:00 - 492 hits        : ░░░░░░░░░░░
-  #   16:00 - 355 hits        : ░░░░░░░░
-  #   17:00 - 213 hits        : ░░░░░
-  #   18:00 - 107 hits        : ░░
+  #  --------------------------------------------------
+  #    7:00 - 330 hits        : =======
+  #    8:00 - 704 hits        : =================
+  #    9:00 - 830 hits        : ====================
+  #   10:00 - 822 hits        : ===================
+  #   11:00 - 823 hits        : ===================
+  #   12:00 - 729 hits        : =================
+  #   13:00 - 614 hits        : ==============
+  #   14:00 - 690 hits        : ================
+  #   15:00 - 492 hits        : ===========
+  #   16:00 - 355 hits        : ========
+  #   17:00 - 213 hits        : =====
+  #   18:00 - 107 hits        : ==
   #   ................
-  class HourlySpread < RequestLogAnalyzer::Tracker::Base
+  class HourlySpread < Base
+
+    attr_reader :hour_frequencies, :first, :last
 
-    attr_reader :first, :last, :request_time_graph
-  
+    # Check if timestamp field is set in the options and prepare the result time graph.
     def prepare
       options[:field] ||= :timestamp
-      @request_time_graph = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+      @hour_frequencies = (0...24).map { 0 }
+      @first, @last = 99999999999999, 0
     end
-              
+
+    # Check if the timestamp in the request and store it.
+    # <tt>request</tt> The request.
     def update(request)
-      request = request.attributes
-      timestamp = request[options[:field]]
+      timestamp = request.first(options[:field])
+      @hour_frequencies[timestamp.to_s[8..9].to_i] +=1
+      @first = timestamp if timestamp < @first
+      @last  = timestamp if timestamp > @last
+    end
 
-      @request_time_graph[timestamp.to_s[8..9].to_i] +=1
-      @first = timestamp if @first.nil? || timestamp < @first
-      @last  = timestamp if @last.nil?  || timestamp > @last
+    # Total amount of requests tracked
+    def total_requests
+      @hour_frequencies.inject(0) { |sum, value| sum + value }
     end
-  
-    def report(output = STDOUT, report_width = 80, color = false)
-      output << "\n"
-      output << "Requests graph - average per day per hour\n"
-      output << green("━" * report_width, color) + "\n"
-      
-      if @request_time_graph == [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+
+
+    # First timestamp encountered
+    def first_timestamp
+      DateTime.parse(@first.to_s, '%Y%m%d%H%M%S') rescue nil
+    end
+
+    # Last timestamp encountered
+    def last_timestamp
+      DateTime.parse(@last.to_s, '%Y%m%d%H%M%S') rescue nil
+    end
+
+    # Difference between last and first timestamp.
+    def timespan
+      last_timestamp - first_timestamp
+    end
+    
+    # Generate an hourly spread report to the given output object.
+    # Any options for the report should have been set during initialize.
+    # <tt>output</tt> The output object
+    def report(output)
+      output.title(title)
+
+      if total_requests == 0
         output << "None found.\n"
         return
       end
 
-      first_date    = DateTime.parse(@first.to_s, '%Y%m%d%H%M%S')
-      last_date     = DateTime.parse(@last.to_s, '%Y%m%d%H%M%S')
-      days          = (@last && @first) ? (last_date - first_date).ceil : 1
-      deviation     = @request_time_graph.max / 20
-      deviation     = 1 if deviation == 0
-      color_cutoff  = 15
-      
-      @request_time_graph.each_with_index do |requests, index|
-        display_chars = requests / deviation
-        request_today = requests / days
-      
-        if display_chars >= color_cutoff
-          display_chars_string = green(('░' * color_cutoff), color) + red(('░' * (display_chars - color_cutoff)), color)
-        else
-          display_chars_string = green(('░' * display_chars), color)
+      days = [1, timespan].max
+      output.table({}, {:align => :right}, {:type => :ratio, :width => :rest, :treshold => 0.15}) do |rows|
+        @hour_frequencies.each_with_index do |requests, index|
+          ratio            = requests.to_f / total_requests.to_f
+          requests_per_day = (requests / days).ceil
+          rows << ["#{index.to_s.rjust(3)}:00", "%d hits/day" % requests_per_day, ratio]
         end
-      
-        output << "#{index.to_s.rjust(3)}:00 - #{(request_today.to_s + ' hits').ljust(15)} : #{display_chars_string}\n"
       end
-      output << "\n"
+    end
+
+    # Returns the title of this tracker for reports
+    def title
+      options[:title] || "Request distribution per hour"
+    end
 
+    # Returns the found frequencies per hour as a hash for YAML exporting
+    def to_yaml_object
+      yaml_object = {}
+      @hour_frequencies.each_with_index do |freq, hour|
+        yaml_object["#{hour}:00 - #{hour+1}:00"] = freq
+      end
+      yaml_object
     end
   end
 end

data/lib/request_log_analyzer/tracker/numeric_value.rb

@@ -0,0 +1,223 @@
+module RequestLogAnalyzer::Tracker
+  
+  class NumericValue < Base
+
+    attr_reader :categories
+
+    # Sets up the numeric value tracker. It will check whether the value and category
+    # options are set that are used to extract and categorize the values during
+    # parsing. Two lambda procedures are created for these tasks
+    def prepare
+      
+      raise "No value field set up for numeric tracker #{self.inspect}" unless options[:value]
+      raise "No categorizer set up for numeric tracker #{self.inspect}" unless options[:category]
+
+      unless options[:multiple]
+        @categorizer = create_lambda(options[:category])
+        @valueizer   = create_lambda(options[:value])
+      end
+
+      @categories = {}
+    end
+
+    # Get the value information from the request and store it in the respective categories.
+    #
+    # If a request can contain multiple usable values for this tracker, the :multiple option 
+    # should be set to true. In this case, all the values and respective categories will be
+    # read from the request using the #every method from the fields given in the :value and 
+    # :category option.
+    #
+    # If the request contains only one suitable value and the :multiple is not set, it will
+    # read the single value and category from the fields provided in the :value and :category
+    # option, or calculate it with any lambda procedure that is assigned to these options. The
+    # request will be passed to procedure as input for the calculation.
+    #
+    # @param [RequestLogAnalyzer::Request] request The request to get the information from.
+    def update(request)
+      if options[:multiple]
+        found_categories = request.every(options[:category])
+        found_values     = request.every(options[:value])
+        raise "Capture mismatch for multiple values in a request" unless found_categories.length == found_values.length
+        
+        found_categories.each_with_index do |cat, index| 
+          update_statistics(cat, found_values[index]) if cat && found_values[index].kind_of?(Numeric)
+        end
+        
+      else
+        category = @categorizer.call(request)
+        value    = @valueizer.call(request)
+        update_statistics(category, value) if value.kind_of?(Numeric) && category
+      end
+    end
+
+    # Block function to build a result table using a provided sorting function.
+    # <tt>output</tt> The output object.
+    # <tt>amount</tt> The number of rows in the report table (default 10).
+    # === Options
+    #  * </tt>:title</tt> The title of the table
+    #  * </tt>:sort</tt> The key to sort on (:hits, :cumulative, :average, :min or :max)
+    def report_table(output, sort, options = {}, &block)
+      output.puts
+      top_categories = output.slice_results(sorted_by(sort))
+      output.with_style(:top_line => true) do
+        output.table(*statistics_header(:title => options[:title], :highlight => sort)) do |rows|
+          top_categories.each { |(cat, info)| rows << statistics_row(cat) }
+        end
+      end
+    end
+
+    # Display a value
+    def display_value(value)
+      return "- " if value.nil?
+      return "0 " if value.zero?
+
+      case [Math.log10(value.abs).floor, 0].max
+        when  0...4  then '%d ' % value
+        when  4...7  then '%dk' % (value / 1000)
+        when  7...10 then '%dM' % (value / 1000_000)
+        when 10...13 then '%dG' % (value / 1000_000_000)
+        when 13...16 then '%dT' % (value / 1000_000_000_000)
+        else              '%dP' % (value / 1000_000_000_000_000)
+      end
+    end
+
+    # Generate a request report to the given output object
+    # By default colulative and average duration are generated.
+    # Any options for the report should have been set during initialize.
+    # <tt>output</tt> The output object
+    def report(output)
+      sortings = output.options[:sort] || [:sum, :mean]
+      sortings.each do |sorting|
+        report_table(output, sorting, :title => "#{title} - by #{sorting}")
+      end
+      
+      if options[:total]
+        output.puts
+        output.puts "#{output.colorize(title, :white, :bold)} - total: " + output.colorize(display_value(sum_overall), :brown, :bold)
+      end
+    end
+
+    # Returns the title of this tracker for reports
+    def title
+      @title ||= begin
+        if options[:title]
+          options[:title]
+        else
+          title_builder = ""
+          title_builder << "#{options[:value]} " if options[:value].kind_of?(Symbol)
+          title_builder << (options[:category].kind_of?(Symbol) ? "per #{options[:category]}" : "per request")
+          title_builder
+        end
+      end
+    end
+
+    # Returns all the categories and the tracked duration as a hash than can be exported to YAML
+    def to_yaml_object
+      return nil if @categories.empty?
+      @categories
+    end
+    
+
+    # Update sthe running calculation of statistics with the newly found numeric value.
+    # <tt>category</tt>:: The category for which to update the running statistics calculations
+    # <tt>number</tt>:: The numeric value to update the calculations with.
+    def update_statistics(category, number)
+      @categories[category] ||= {:hits => 0, :sum => 0, :mean => 0.0, :sum_of_squares => 0.0, :min => number, :max => number }
+      delta = number - @categories[category][:mean]
+
+      @categories[category][:hits]           += 1
+      @categories[category][:mean]           += (delta / @categories[category][:hits])
+      @categories[category][:sum_of_squares] += delta * (number - @categories[category][:mean])
+      @categories[category][:sum]            += number
+      @categories[category][:min]             = number if number < @categories[category][:min]
+      @categories[category][:max]             = number if number > @categories[category][:max]
+    end
+
+    # Get the number of hits of a specific category.
+    # <tt>cat</tt> The category
+    def hits(cat)
+      @categories[cat][:hits]
+    end
+
+    # Get the total duration of a specific category.
+    # <tt>cat</tt> The category
+    def sum(cat)
+      @categories[cat][:sum]
+    end
+
+    # Get the minimal duration of a specific category.
+    # <tt>cat</tt> The category
+    def min(cat)
+      @categories[cat][:min]
+    end
+
+    # Get the maximum duration of a specific category.
+    # <tt>cat</tt> The category
+    def max(cat)
+      @categories[cat][:max]
+    end
+
+    # Get the average duration of a specific category.
+    # <tt>cat</tt> The category
+    def mean(cat)
+      @categories[cat][:mean]
+    end
+
+    # Get the standard deviation of the duration of a specific category.
+    # <tt>cat</tt> The category
+    def stddev(cat)
+      Math.sqrt(variance(cat))
+    end
+
+    # Get the variance of the duration of a specific category.
+    # <tt>cat</tt> The category
+    def variance(cat)
+      return 0.0 if @categories[cat][:hits] <= 1
+      (@categories[cat][:sum_of_squares] / (@categories[cat][:hits] - 1))
+    end
+
+    # Get the average duration of a all categories.
+    def mean_overall
+      sum_overall / hits_overall
+    end
+
+    # Get the cumlative duration of a all categories.
+    def sum_overall
+      @categories.inject(0.0) { |sum, (name, cat)| sum + cat[:sum] }
+    end
+
+    # Get the total hits of a all categories.
+    def hits_overall
+      @categories.inject(0) { |sum, (name, cat)| sum + cat[:hits] }
+    end
+
+    # Return categories sorted by a given key.
+    # <tt>by</tt> The key to sort on. This parameter can be omitted if a sorting block is provided instead
+    def sorted_by(by = nil)
+      if block_given?
+        categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }
+      else
+        categories.sort { |a, b| send(by, b[0]) <=> send(by, a[0]) }
+      end
+    end
+
+    # Returns the column header for a statistics table to report on the statistics result
+    def statistics_header(options)
+      [
+        {:title => options[:title], :width => :rest},
+        {:title => 'Hits',   :align => :right, :highlight => (options[:highlight] == :hits),   :min_width => 4},
+        {:title => 'Sum',    :align => :right, :highlight => (options[:highlight] == :sum),    :min_width => 6},
+        {:title => 'Mean',   :align => :right, :highlight => (options[:highlight] == :mean),   :min_width => 6},
+        {:title => 'StdDev', :align => :right, :highlight => (options[:highlight] == :stddev), :min_width => 6},
+        {:title => 'Min',    :align => :right, :highlight => (options[:highlight] == :min),    :min_width => 6},
+        {:title => 'Max',    :align => :right, :highlight => (options[:highlight] == :max),    :min_width => 6}
+      ]
+    end
+
+    # Returns a row of statistics information for a report table, given a category
+    def statistics_row(cat)
+      [cat, hits(cat), display_value(sum(cat)), display_value(mean(cat)), display_value(stddev(cat)),
+                display_value(min(cat)), display_value(max(cat))]
+    end
+  end
+end