ngmoco-request-log-analyzer 1.4.2

data/lib/request_log_analyzer/tracker.rb

@@ -0,0 +1,206 @@
+module RequestLogAnalyzer::Tracker
+
+  # const_missing: this function is used to load subclasses in the RequestLogAnalyzer::Track namespace.
+  # It will automatically load the required file based on the class name
+  def self.const_missing(const)
+    RequestLogAnalyzer::load_default_class_file(self, const)
+  end
+
+  # Base Tracker class. All other trackers inherit from this class
+  #
+  # Accepts the following options:
+  # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+  # * <tt>:output</tt> Direct output here (defaults to STDOUT)
+  # * <tt>:unless</tt> Proc that has to return nil for a request to be passed to the tracker.
+  #
+  # For example :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
+  class Base
+
+    attr_reader :options
+
+    # Initialize the class
+    # Note that the options are only applicable if should_update? is not overwritten
+    # by the inheriting class.
+    #
+    # === Options
+    # * <tt>:if</tt> Handle request if this proc is true for the handled request.
+    # * <tt>:unless</tt> Handle request if this proc is false for the handled request.
+    # * <tt>:line_type</tt> Line type this tracker will accept.
+    def initialize(options ={})
+      @options = options
+      setup_should_update_checks!
+    end
+
+    # Sets up the tracker's should_update? checks.
+    def setup_should_update_checks!
+      @should_update_checks = []
+      @should_update_checks.push( lambda { |request| request.has_line_type?(options[:line_type]) } ) if options[:line_type]
+      @should_update_checks.push(options[:if]) if options[:if].respond_to?(:call)
+      @should_update_checks.push( lambda { |request| request[options[:if]] }) if options[:if].kind_of?(Symbol)
+      @should_update_checks.push( lambda { |request| !options[:unless].call(request) }) if options[:unless].respond_to?(:call)
+      @should_update_checks.push( lambda { |request| !request[options[:unless]] }) if options[:unless].kind_of?(Symbol)
+    end
+    
+    # Creates a lambda expression to return a static field from a request. If the 
+    # argument already is a lambda exprssion, it will simply return the argument.
+    def create_lambda(arg)
+      case arg
+      when Proc   then arg
+      when Symbol then lambda { |request| request[arg] }
+      else raise "Canot create a lambda expression from this argument: #{arg.inspect}!"
+      end
+    end
+
+    # Hook things that need to be done before running here.
+    def prepare
+    end
+
+    # Will be called with each request.
+    # <tt>request</tt> The request to track data in.
+    def update(request)
+    end
+
+    # Hook things that need to be done after running here.
+    def finalize
+    end
+
+    # Determine if we should run the update function at all.
+    # Usually the update function will be heavy, so a light check is done here
+    # determining if we need to call update at all.
+    #
+    # Default this checks if defined:
+    #  * :line_type is also in the request hash.
+    #  * :if is true for this request.
+    #  * :unless if false for this request
+    #
+    # <tt>request</tt> The request object.
+    def should_update?(request)
+      @should_update_checks.all? { |c| c.call(request) }
+    end
+
+    # Hook report generation here.
+    # Defaults to self.inspect
+    # <tt>output</tt> The output object the report will be passed to.
+    def report(output)
+      output << self.inspect
+      output << "\n"
+    end
+
+    # The title of this tracker. Used for reporting.
+    def title
+      self.class.to_s
+    end
+
+    # This method is called by RequestLogAnalyzer::Aggregator:Summarizer to retrieve an
+    # object with all the results of this tracker, that can be dumped to YAML format.
+    def to_yaml_object
+      nil
+    end
+  end
+  
+  module StatisticsTracking
+    
+    # Update sthe running calculation of statistics with the newly found numeric value.
+    # <tt>category</tt>:: The category for which to update the running statistics calculations
+    # <tt>number</tt>:: The numeric value to update the calculations with.
+    def update_statistics(category, number)
+      @categories[category] ||= {:hits => 0, :sum => 0, :mean => 0.0, :sum_of_squares => 0.0, :min => number, :max => number }
+      delta = number - @categories[category][:mean]
+
+      @categories[category][:hits]           += 1
+      @categories[category][:mean]           += (delta / @categories[category][:hits])
+      @categories[category][:sum_of_squares] += delta * (number - @categories[category][:mean])
+      @categories[category][:sum]            += number
+      @categories[category][:min]             = number if number < @categories[category][:min]
+      @categories[category][:max]             = number if number > @categories[category][:max]
+    end
+    
+    # Get the number of hits of a specific category.
+    # <tt>cat</tt> The category
+    def hits(cat)
+      @categories[cat][:hits]
+    end
+
+    # Get the total duration of a specific category.
+    # <tt>cat</tt> The category
+    def sum(cat)
+      @categories[cat][:sum]
+    end
+
+    # Get the minimal duration of a specific category.
+    # <tt>cat</tt> The category
+    def min(cat)
+      @categories[cat][:min]
+    end
+
+    # Get the maximum duration of a specific category.
+    # <tt>cat</tt> The category
+    def max(cat)
+      @categories[cat][:max]
+    end
+
+    # Get the average duration of a specific category.
+    # <tt>cat</tt> The category
+    def mean(cat)
+      @categories[cat][:mean]
+    end
+
+    # Get the standard deviation of the duration of a specific category.
+    # <tt>cat</tt> The category
+    def stddev(cat)
+      Math.sqrt(variance(cat))
+    end
+
+    # Get the variance of the duration of a specific category.
+    # <tt>cat</tt> The category
+    def variance(cat)
+      return 0.0 if @categories[cat][:hits] <= 1
+      (@categories[cat][:sum_of_squares] / (@categories[cat][:hits] - 1))
+    end
+
+    # Get the average duration of a all categories.
+    def mean_overall
+      sum_overall / hits_overall
+    end
+
+    # Get the cumlative duration of a all categories.
+    def sum_overall
+      @categories.inject(0.0) { |sum, (name, cat)| sum + cat[:sum] }
+    end
+
+    # Get the total hits of a all categories.
+    def hits_overall
+      @categories.inject(0) { |sum, (name, cat)| sum + cat[:hits] }
+    end
+    
+    # Return categories sorted by a given key.
+    # <tt>by</tt> The key to sort on. This parameter can be omitted if a sorting block is provided instead
+    def sorted_by(by = nil)
+      if block_given?
+        categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }
+      else
+        categories.sort { |a, b| send(by, b[0]) <=> send(by, a[0]) }
+      end
+    end
+    
+    # Returns the column header for a statistics table to report on the statistics result
+    def statistics_header(options)
+      [
+        {:title => options[:title], :width => :rest},
+        {:title => 'Hits',   :align => :right, :highlight => (options[:highlight] == :hits),   :min_width => 4},
+        {:title => 'Sum',    :align => :right, :highlight => (options[:highlight] == :sum),    :min_width => 6},
+        {:title => 'Mean',   :align => :right, :highlight => (options[:highlight] == :mean),   :min_width => 6},
+        {:title => 'StdDev', :align => :right, :highlight => (options[:highlight] == :stddev), :min_width => 6},
+        {:title => 'Min',    :align => :right, :highlight => (options[:highlight] == :min),    :min_width => 6},
+        {:title => 'Max',    :align => :right, :highlight => (options[:highlight] == :max),    :min_width => 6}
+      ]
+    end
+    
+    # Returns a row of statistics information for a report table, given a category
+    def statistics_row(cat)
+      [cat, hits(cat), display_value(sum(cat)), display_value(mean(cat)), display_value(stddev(cat)),
+                display_value(min(cat)), display_value(max(cat))]
+    end
+  end
+end

data/lib/request_log_analyzer/tracker/duration.rb

@@ -0,0 +1,104 @@
+module RequestLogAnalyzer::Tracker
+
+  # Analyze the duration of a specific attribute
+  #
+  # === Options
+  # * <tt>:category</tt> Proc that handles request categorization for given fileformat (REQUEST_CATEGORIZER)
+  # * <tt>:duration</tt> The field containing the duration in the request hash.
+  # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+  # * <tt>:title</tt> Title do be displayed above the report
+  # * <tt>:unless</tt> Handle request if this proc is false for the handled request.
+  #
+  # The items in the update request hash are set during the creation of the Duration tracker.
+  #
+  # Example output:
+  #  Request duration - top 20 by cumulative time   |    Hits |      Sum. |      Avg.
+  #  ---------------------------------------------------------------------------------
+  #  EmployeeController#show.html [GET]             |    4742 |  4922.56s |     1.04s
+  #  EmployeeController#update.html [POST]          |    4647 |  2731.23s |     0.59s
+  #  EmployeeController#index.html [GET]            |    5802 |  1477.32s |     0.25s
+  #  .............
+  class Duration < Base
+
+    include RequestLogAnalyzer::Tracker::StatisticsTracking
+
+    attr_reader :categories
+
+    # Check if duration and catagory option have been received,
+    def prepare
+      raise "No duration field set up for category tracker #{self.inspect}" unless options[:duration]
+      raise "No categorizer set up for duration tracker #{self.inspect}"    unless options[:category]
+
+      unless options[:multiple]
+        @categorizer  = create_lambda(options[:category])
+        @durationizer = create_lambda(options[:duration])
+      end
+      
+      @categories   = {}
+    end
+
+    # Get the duration information fron the request and store it in the different categories.
+    # <tt>request</tt> The request.
+    def update(request)
+      if options[:multiple]
+        found_categories = request.every(options[:category])
+        found_durations  = request.every(options[:duration])
+        raise "Capture mismatch for multiple values in a request" unless found_categories.length == found_durations.length
+        found_categories.each_with_index { |cat, index| update_statistics(cat, found_durations[index]) }
+      else
+        category = @categorizer.call(request)
+        duration = @durationizer.call(request)
+        update_statistics(category, duration) if duration.kind_of?(Numeric) && category
+      end
+    end
+
+    # Block function to build a result table using a provided sorting function.
+    # <tt>output</tt> The output object.
+    # <tt>amount</tt> The number of rows in the report table (default 10).
+    # === Options
+    #  * </tt>:title</tt> The title of the table
+    #  * </tt>:sort</tt> The key to sort on (:hits, :cumulative, :average, :min or :max)
+    def report_table(output, sort, options = {}, &block)
+      output.puts
+      top_categories = output.slice_results(sorted_by(sort))
+      output.with_style(:top_line => true) do
+        output.table(*statistics_header(:title => options[:title], :highlight => sort)) do |rows|
+          top_categories.each { |(cat, info)| rows << statistics_row(cat) }
+        end
+      end
+    end
+
+    # Display a duration
+    def display_value(time)
+      case time
+      when nil       then '-'
+      when 0...60    then "%0.02fs" % time
+      when 60...3600 then "%dm%02ds" % [time / 60, (time % 60).round]
+      else                "%dh%02dm%02ds" % [time / 3600, (time % 3600) / 60, (time % 60).round]
+      end
+    end
+
+    # Generate a request duration report to the given output object
+    # By default colulative and average duration are generated.
+    # Any options for the report should have been set during initialize.
+    # <tt>output</tt> The output object
+    def report(output)
+      sortings = output.options[:sort] || [:sum, :mean]
+      sortings.each do |sorting|
+        report_table(output, sorting, :title => "#{title} - by #{sorting}")
+      end
+    end
+
+    # Returns the title of this tracker for reports
+    def title
+      options[:title]  || 'Request duration'
+    end
+
+    # Returns all the categories and the tracked duration as a hash than can be exported to YAML
+    def to_yaml_object
+      return nil if @categories.empty?
+      @categories
+    end
+  end
+end

data/lib/request_log_analyzer/tracker/frequency.rb

@@ -0,0 +1,95 @@
+module RequestLogAnalyzer::Tracker
+
+  # Catagorize requests by frequency.
+  # Count and analyze requests for a specific attribute
+  #
+  # === Options
+  # * <tt>:category</tt> Proc that handles the request categorization.
+  # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+  # * <tt>:nils</tt> Track undetermined methods.
+  # * <tt>:title</tt> Title do be displayed above the report.
+  # * <tt>:unless</tt> Proc that has to return nil for a request to be passed to the tracker.
+  #
+  # The items in the update request hash are set during the creation of the Duration tracker.
+  #
+  # Example output:
+  #  HTTP methods
+  #  ----------------------------------------------------------------------
+  #  GET    |  22248 hits (46.2%) |=================
+  #  PUT    |  13685 hits (28.4%) |===========
+  #  POST   |  11662 hits (24.2%) |=========
+  #  DELETE |    512 hits (1.1%)  |
+  class Frequency < Base
+
+    attr_reader :categories
+
+    # Check if categories are set up
+    def prepare
+      raise "No categorizer set up for category tracker #{self.inspect}" unless options[:category]
+      @categorizer = create_lambda(options[:category])
+      
+      # Initialize the categories. Use the list of category names to 
+      @categories = {}
+      options[:all_categories].each { |cat| @categories[cat] = 0 } if options[:all_categories].kind_of?(Enumerable)
+    end
+
+    # Check HTTP method of a request and store that in the categories hash.
+    # <tt>request</tt> The request.
+    def update(request)
+      cat = @categorizer.call(request)
+      if cat || options[:nils]
+        @categories[cat] ||= 0
+        @categories[cat] += 1
+      end
+    end
+
+    # Return the amount of times a HTTP method has been encountered
+    # <tt>cat</tt> The HTTP method (:get, :put, :post or :delete)
+    def frequency(cat)
+      categories[cat] || 0
+    end
+
+    # Return the overall frequency
+    def overall_frequency
+      categories.inject(0) { |carry, item| carry + item[1] }
+    end
+
+    # Return the methods sorted by frequency
+    def sorted_by_frequency
+      @categories.sort { |a, b| b[1] <=> a[1] }
+    end
+
+    # Generate a HTTP method frequency report to the given output object.
+    # Any options for the report should have been set during initialize.
+    # <tt>output</tt> The output object
+    def report(output)
+      output.title(options[:title]) if options[:title]
+
+      if @categories.empty?
+        output << "None found.\n"
+      else
+        sorted_categories = output.slice_results(sorted_by_frequency)
+        total_hits        = overall_frequency
+
+        output.table({:align => :left}, {:align => :right }, {:align => :right}, {:type => :ratio, :width => :rest}) do |rows|
+          sorted_categories.each do |(cat, count)|
+            rows << [cat, "#{count} hits", '%0.1f%%' % ((count.to_f / total_hits.to_f) * 100.0), (count.to_f / total_hits.to_f)]
+          end
+        end
+
+      end
+    end
+
+    # Returns a hash with the categories of every category that can be exported to YAML
+    def to_yaml_object
+      return nil if @categories.empty?
+      @categories
+    end
+
+    # Returns the title of this tracker for reports
+    def title
+      options[:title] || 'Request frequency'
+    end
+  end
+end

data/lib/request_log_analyzer/tracker/hourly_spread.rb

@@ -0,0 +1,107 @@
+module RequestLogAnalyzer::Tracker
+
+  # Determines the average hourly spread of the parsed requests.
+  # This spread is shown in a graph form.
+  #
+  # Accepts the following options:
+  # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+  # * <tt>:output</tt> Direct output here (defaults to STDOUT)
+  # * <tt>:unless</tt> Proc that has to return nil for a request to be passed to the tracker.
+  #
+  # Expects the following items in the update request hash
+  # * <tt>:timestamp</tt> in YYYYMMDDHHMMSS format.
+  #
+  # Example output:
+  #  Requests graph - average per day per hour
+  #  --------------------------------------------------
+  #    7:00 - 330 hits        : =======
+  #    8:00 - 704 hits        : =================
+  #    9:00 - 830 hits        : ====================
+  #   10:00 - 822 hits        : ===================
+  #   11:00 - 823 hits        : ===================
+  #   12:00 - 729 hits        : =================
+  #   13:00 - 614 hits        : ==============
+  #   14:00 - 690 hits        : ================
+  #   15:00 - 492 hits        : ===========
+  #   16:00 - 355 hits        : ========
+  #   17:00 - 213 hits        : =====
+  #   18:00 - 107 hits        : ==
+  #   ................
+  class HourlySpread < Base
+
+    attr_reader :hour_frequencies, :first, :last
+
+    # Check if timestamp field is set in the options and prepare the result time graph.
+    def prepare
+      options[:field] ||= :timestamp
+      @hour_frequencies = (0...24).map { 0 }
+      @first, @last = 99999999999999, 0
+    end
+
+    # Check if the timestamp in the request and store it.
+    # <tt>request</tt> The request.
+    def update(request)
+      timestamp = request.first(options[:field])
+      @hour_frequencies[timestamp.to_s[8..9].to_i] +=1
+      @first = timestamp if timestamp < @first
+      @last  = timestamp if timestamp > @last
+    end
+
+    # Total amount of requests tracked
+    def total_requests
+      @hour_frequencies.inject(0) { |sum, value| sum + value }
+    end
+
+
+    # First timestamp encountered
+    def first_timestamp
+      DateTime.parse(@first.to_s, '%Y%m%d%H%M%S') rescue nil
+    end
+
+    # Last timestamp encountered
+    def last_timestamp
+      DateTime.parse(@last.to_s, '%Y%m%d%H%M%S') rescue nil
+    end
+
+    # Difference between last and first timestamp.
+    def timespan
+      last_timestamp - first_timestamp
+    end
+    
+    # Generate an hourly spread report to the given output object.
+    # Any options for the report should have been set during initialize.
+    # <tt>output</tt> The output object
+    def report(output)
+      output.title(title)
+
+      if total_requests == 0
+        output << "None found.\n"
+        return
+      end
+
+      days = [1, timespan].max
+      output.table({}, {:align => :right}, {:type => :ratio, :width => :rest, :treshold => 0.15}) do |rows|
+        @hour_frequencies.each_with_index do |requests, index|
+          ratio            = requests.to_f / total_requests.to_f
+          requests_per_day = (requests / days).ceil
+          rows << ["#{index.to_s.rjust(3)}:00", "%d hits/day" % requests_per_day, ratio]
+        end
+      end
+    end
+
+    # Returns the title of this tracker for reports
+    def title
+      options[:title] || "Request distribution per hour"
+    end
+
+    # Returns the found frequencies per hour as a hash for YAML exporting
+    def to_yaml_object
+      yaml_object = {}
+      @hour_frequencies.each_with_index do |freq, hour|
+        yaml_object["#{hour}:00 - #{hour+1}:00"] = freq
+      end
+      yaml_object
+    end
+  end
+end