wvanbergen-request-log-analyzer 1.0.0 → 1.0.1

data/lib/cli/progressbar.rb

@@ -0,0 +1,236 @@
+#
+# Ruby/ProgressBar - a text progress bar library
+#
+# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms
+# of Ruby's license.
+#
+
+class ProgressBar
+  VERSION = "0.9"
+
+  def initialize (title, total, out = STDERR)
+    @title = title
+    @total = total
+    @out = out
+    @terminal_width = 80
+    @bar_mark = '='
+    @current = 0
+    @previous = 0
+    @finished_p = false
+    @start_time = Time.now
+    @previous_time = @start_time
+    @title_width = 24
+    @format = "%-#{@title_width}s %3d%% %s %s"
+    @format_arguments = [:title, :percentage, :bar, :stat]
+    clear
+    show
+  end
+  attr_reader   :title
+  attr_reader   :current
+  attr_reader   :total
+  attr_accessor :start_time
+
+  private
+  def fmt_bar
+    bar_width = do_percentage * @terminal_width / 100
+    sprintf("[%s%s]", 
+            @bar_mark * bar_width, 
+            " " *  (@terminal_width - bar_width))
+  end
+
+  def fmt_percentage
+    do_percentage
+  end
+
+  def fmt_stat
+    if @finished_p then elapsed else eta end
+  end
+
+  def fmt_stat_for_file_transfer
+    if @finished_p then 
+      sprintf("%s %s %s", bytes, transfer_rate, elapsed)
+    else 
+      sprintf("%s %s %s", bytes, transfer_rate, eta)
+    end
+  end
+
+  def fmt_title
+    @title[0,(@title_width - 1)] + ":"
+  end
+
+  def convert_bytes (bytes)
+    if bytes < 1024
+      sprintf("%6dB", bytes)
+    elsif bytes < 1024 * 1000 # 1000kb
+      sprintf("%5.1fKB", bytes.to_f / 1024)
+    elsif bytes < 1024 * 1024 * 1000  # 1000mb
+      sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
+    else
+      sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+    end
+  end
+
+  def transfer_rate
+    bytes_per_second = @current.to_f / (Time.now - @start_time)
+    sprintf("%s/s", convert_bytes(bytes_per_second))
+  end
+
+  def bytes
+    convert_bytes(@current)
+  end
+
+  def format_time (t)
+    t = t.to_i
+    sec = t % 60
+    min  = (t / 60) % 60
+    hour = t / 3600
+    sprintf("%02d:%02d:%02d", hour, min, sec);
+  end
+
+  # ETA stands for Estimated Time of Arrival.
+  def eta
+    if @current == 0
+      "ETA:  --:--:--"
+    else
+      elapsed = Time.now - @start_time
+      eta = elapsed * @total / @current - elapsed;
+      sprintf("ETA:  %s", format_time(eta))
+    end
+  end
+
+  def elapsed
+    elapsed = Time.now - @start_time
+    sprintf("Time: %s", format_time(elapsed))
+  end
+  
+  def eol
+    if @finished_p then "\n" else "\r" end
+  end
+
+  def do_percentage
+    if @total.zero?
+      100
+    else
+      @current  * 100 / @total
+    end
+  end
+
+  def get_width
+    # FIXME: I don't know how portable it is.
+    default_width = 80
+    begin
+      tiocgwinsz = 0x5413
+      data = [0, 0, 0, 0].pack("SSSS")
+      if @out.ioctl(tiocgwinsz, data) >= 0 then
+        rows, cols, xpixels, ypixels = data.unpack("SSSS")
+        if cols >= 0 then cols else default_width end
+      else
+        default_width
+      end
+    rescue Exception
+      default_width
+    end
+  end
+
+  def show
+    arguments = @format_arguments.map {|method| 
+      method = sprintf("fmt_%s", method)
+      send(method)
+    }
+    line = sprintf(@format, *arguments)
+
+    width = get_width
+    if line.length == width - 1 
+      @out.print(line + eol)
+      @out.flush
+    elsif line.length >= width
+      @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+      if @terminal_width == 0 then @out.print(line + eol) else show end
+    else # line.length < width - 1
+      @terminal_width += width - line.length + 1
+      show
+    end
+    @previous_time = Time.now
+  end
+
+  def show_if_needed
+    if @total.zero?
+      cur_percentage = 100
+      prev_percentage = 0
+    else
+      cur_percentage  = (@current  * 100 / @total).to_i
+      prev_percentage = (@previous * 100 / @total).to_i
+    end
+
+    # Use "!=" instead of ">" to support negative changes
+    if cur_percentage != prev_percentage || 
+        Time.now - @previous_time >= 1 || @finished_p
+      show
+    end
+  end
+
+  public
+  def clear
+    @out.print "\r"
+    @out.print(" " * (get_width - 1))
+    @out.print "\r"
+  end
+
+  def finish
+    @current = @total
+    @finished_p = true
+    show
+  end
+
+  def finished?
+    @finished_p
+  end
+
+  def file_transfer_mode
+    @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
+  end
+
+  def format= (format)
+    @format = format
+  end
+
+  def format_arguments= (arguments)
+    @format_arguments = arguments
+  end
+
+  def halt
+    @finished_p = true
+    show
+  end
+
+  def inc (step = 1)
+    @current += step
+    @current = @total if @current > @total
+    show_if_needed
+    @previous = @current
+  end
+
+  def set (count)
+    count = 0 if count < 0 
+    count = @total if count > @total
+    
+    @current = count
+    show_if_needed
+    @previous = @current
+  end
+
+  def inspect
+    "#<ProgressBar:#{@current}/#{@total}>"
+  end
+end
+
+class ReversedProgressBar < ProgressBar
+  def do_percentage
+    100 - super
+  end
+end
+

data/lib/request_log_analyzer/aggregator/base.rb

@@ -0,0 +1,51 @@
+module RequestLogAnalyzer::Aggregator
+
+  # The base class of an aggregator. This class provides the interface to which
+  # every aggregator should comply (by simply subclassing this class).
+  #
+  # When building an aggregator, do not forget that RequestLogAnalyzer can run in
+  # single line mode or in combined requests mode. Make sure your aggregator can 
+  # handle both cases, or raise an exception if RLA is rnning in the wrong mode.
+  # Calling options[:combined_requests] tells you if RLA is running in combined 
+  # requests mode, otherwise it is running in single line mode.
+  class Base
+    
+    include RequestLogAnalyzer::FileFormat::Awareness
+    
+    attr_reader :options
+    attr_reader :source
+
+    # Intializes a new RequestLogAnalyzer::Aggregator::Base instance
+    # It will include the specific file format module.
+    def initialize(source, options = {})
+      @source = source
+      self.register_file_format(source.file_format)
+      @options = options
+    end
+
+    # The prepare function is called just before parsing starts. This function 
+    # can be used to initialie variables, etc.
+    def prepare
+    end
+    
+    # The aggregate function is called for every request.
+    # Implement the aggregating functionality in this method
+    def aggregate(request)
+    end
+    
+    # The finalize function is called after all sources are parsed and no more
+    # requests will be passed to the aggregator
+    def finalize
+    end
+    
+    # The warning method is called if the parser eits a warning.
+    def warning(type, message, lineno)
+    end    
+    
+    # The report function is called at the end. Implement any result reporting
+    # in this function.
+    def report(output = STDOUT, report_width = 80, color = false)
+    end
+
+  end
+end

data/lib/request_log_analyzer/aggregator/database.rb

@@ -0,0 +1,97 @@
+require 'rubygems'
+require 'activerecord'
+
+module RequestLogAnalyzer::Aggregator
+
+  class Database < Base
+
+    attr_reader :request_id
+
+    def prepare
+      ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => options[:database])
+
+      File.unlink(options[:database]) if File.exist?(options[:database])
+      create_database_schema!
+      
+      @request_id = 0
+    end
+    
+    def aggregate(request)
+      @request_id += 1
+      
+      request.lines.each do |line|
+        class_name = "#{line[:line_type]}_line".camelize #split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')
+        
+        attributes = line.reject { |k, v| [:line_type].include?(k) }
+        attributes[:request_id] = @request_id if options[:combined_requests]  
+        file_format.class.const_get(class_name).create!(attributes)
+      end
+    rescue SQLite3::SQLException => e
+      raise Interrupt, e.message
+    end
+    
+    def finalize
+      ActiveRecord::Base.remove_connection
+    end
+    
+    def warning(type, message, lineno)
+      file_format.class::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
+    end
+    
+    def report(output = STDOUT, report_width = 80, color = false)
+      output << "\n"
+      output << green("━" * report_width, color) + "\n"
+      output <<  "A database file has been created with all parsed request information.\n"
+      output <<  "To execute queries on this database, run the following command:\n"
+      output <<  "  $ sqlite3 #{options[:database]}\n"
+      output << "\n"
+    end
+    
+    protected 
+    
+    def create_database_table(name, definition)
+      ActiveRecord::Migration.verbose = options[:debug]
+      ActiveRecord::Migration.create_table("#{name}_lines") do |t|
+        t.column(:request_id, :integer) #if options[:combined_requests]
+        t.column(:lineno, :integer)
+        definition.captures.each do |capture|
+          t.column(capture[:name], column_type(capture))
+        end
+      end
+    end
+
+    def create_warning_table_and_class
+      ActiveRecord::Migration.verbose = options[:debug]
+      ActiveRecord::Migration.create_table("warnings") do |t|
+        t.string  :warning_type, :limit => 30, :null => false
+        t.string  :message
+        t.integer :lineno          
+      end    
+      
+      file_format.class.const_set('Warning', Class.new(ActiveRecord::Base)) unless file_format.class.const_defined?('Warning')
+    end
+
+    def create_activerecord_class(name, definition)
+      class_name = "#{name}_line".camelize
+      file_format.class.const_set(class_name, Class.new(ActiveRecord::Base)) unless file_format.class.const_defined?(class_name)
+    end
+
+    def create_database_schema!
+      file_format.line_definitions.each do |name, definition|
+        create_database_table(name, definition)
+        create_activerecord_class(name, definition)
+      end
+      
+      create_warning_table_and_class
+    end
+    
+    def column_type(capture)
+      case capture[:type]
+      when :sec;   :double
+      when :msec;  :double
+      when :float; :double
+      else         capture[:type]
+      end
+    end
+  end
+end

data/lib/request_log_analyzer/aggregator/echo.rb

@@ -0,0 +1,25 @@
+module RequestLogAnalyzer::Aggregator
+
+  class Echo < Base
+    
+    def prepare
+      @warnings = ""
+    end
+    
+    def aggregate(request)
+      puts "\nRequest: " + request.inspect
+    end
+    
+    def warning(type, message, lineno)
+      @warnings << "WARNING #{type.inspect} on line #{lineno}: #{message}\n"
+    end
+    
+    def report(output=STDOUT, report_width = 80, color = false)
+      output << "\n"
+      output << "Warnings during parsing:\n"
+      output << green("━" * report_width, color) + "\n"
+      output << @warnings + "\n"
+    end
+
+  end
+end

data/lib/request_log_analyzer/aggregator/summarizer.rb

@@ -0,0 +1,116 @@
+require File.dirname(__FILE__) + '/../tracker/base'
+
+module RequestLogAnalyzer::Aggregator
+
+  class Summarizer < Base
+    
+    class Definer
+      
+      attr_reader :trackers
+      
+      def initialize
+        @trackers = []
+      end
+      
+      def method_missing(tracker_method, *args)
+        track(tracker_method, args.first)
+      end
+      
+      def category(category_field, options = {})
+        if category_field.kind_of?(Symbol)
+          track(:category, options.merge(:category => category_field))
+        elsif category_field.kind_of?(Hash)
+          track(:category, category_field.merge(options))
+        end
+      end
+      
+      def duration(duration_field, options = {})
+        if duration_field.kind_of?(Symbol)
+          track(:duration, options.merge(:duration => duration_field))
+        elsif duration_field.kind_of?(Hash)
+          track(:duration, duration_field.merge(options))        
+        end
+      end      
+      
+      def track(tracker_klass, options = {})
+        require "#{File.dirname(__FILE__)}/../tracker/#{tracker_klass}"
+        tracker_klass = RequestLogAnalyzer::Tracker.const_get(tracker_klass.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')) if tracker_klass.kind_of?(Symbol)
+        @trackers << tracker_klass.new(options)
+      end
+    end
+    
+    attr_reader :trackers
+    attr_reader :warnings_encountered
+    
+    def initialize(source, options = {})
+      super(source, options)
+      @warnings_encountered = {}
+      @trackers = source.file_format.report_trackers
+      setup
+    end
+    
+    def setup
+    end
+    
+    def prepare
+      raise "No trackers set up in Summarizer!" if @trackers.nil? || @trackers.empty?
+      @trackers.each { |tracker| tracker.prepare }
+      end
+    
+    def aggregate(request)
+      @trackers.each do |tracker|
+        tracker.update(request) if tracker.should_update?(request)
+      end
+    end
+    
+    def finalize
+      @trackers.each { |tracker| tracker.finalize }
+    end
+       
+    def report(output=STDOUT, report_width = 80, color = false)
+      report_header(output, report_width, color)
+      if source.parsed_requests - source.skipped_requests > 0
+        @trackers.each { |tracker| tracker.report(output, report_width, color) }
+      else
+        output << "\n"
+        output << "There were no requests analyzed.\n"
+      end
+      report_footer(output, report_width, color)
+    end
+    
+    def report_header(output=STDOUT, report_width = 80, color = false)
+      output << "Request summary\n"
+      output << green("━" * report_width, color) + "\n"
+      output << "Parsed lines:         #{green(source.parsed_lines, color)}\n"
+      output << "Parsed requests:      #{green(source.parsed_requests, color)}\n"  if options[:combined_requests]
+      output << "Skipped requests:     #{green(source.skipped_requests, color)}\n" if source.skipped_requests > 0
+      if has_warnings?
+        output <<  "Warnings:             " + @warnings_encountered.map { |(key, value)| "#{key.inspect}: #{blue(value, color)}" }.join(', ') + "\n"
+      end
+      output << "\n"
+    end
+    
+    def report_footer(output=STDOUT, report_width = 80, color = false)
+      output << "\n" 
+      if has_serious_warnings?      
+        output << green("━" * report_width, color) + "\n"
+        output << "Multiple warnings were encountered during parsing. Possibly, your logging " + "\n"
+        output << "is not setup correctly. Visit this website for logging configuration tips:" + "\n"
+        output <<  blue("http://github.com/wvanbergen/request-log-analyzer/wikis/configure-logging", color) + "\n"
+      end
+    end
+    
+    def has_warnings?
+       @warnings_encountered.inject(0) { |result, (key, value)| result += value } > 0
+    end
+    
+    def has_serious_warnings?
+      @warnings_encountered.inject(0) { |result, (key, value)| result += value } > 10
+    end
+    
+    def warning(type, message, lineno)
+      @warnings_encountered[type] ||= 0
+      @warnings_encountered[type] += 1
+    end
+  end
+end