request-log-analyzer 1.0.2 → 1.6.3

data/lib/cli/progressbar.rb

@@ -7,230 +7,207 @@
 #
 # You can redistribute it and/or modify it under the terms
 # of Ruby's license.
-#
 
-class ProgressBar
-  VERSION = "0.9"
-
-  def initialize (title, total, out = STDERR)
-    @title = title
-    @total = total
-    @out = out
-    @terminal_width = 80
-    @bar_mark = '='
-    @current = 0
-    @previous = 0
-    @finished_p = false
-    @start_time = Time.now
-    @previous_time = @start_time
-    @title_width = 24
-    @format = "%-#{@title_width}s %3d%% %s %s"
-    @format_arguments = [:title, :percentage, :bar, :stat]
-    clear
-    show
-  end
-  attr_reader   :title
-  attr_reader   :current
-  attr_reader   :total
-  attr_accessor :start_time
-
-  private
-  def fmt_bar
-    bar_width = do_percentage * @terminal_width / 100
-    sprintf("[%s%s]", 
-            @bar_mark * bar_width, 
-            " " *  (@terminal_width - bar_width))
-  end
+module CommandLine
+  class ProgressBar
+    VERSION = "0.9"
+
+    def initialize (title, total, out = STDERR)
+      @title = title
+      @total = total
+      @out = out
+      @terminal_width = 80
+      @bar_mark = '='
+      @current = 0
+      @previous = 0
+      @finished_p = false
+      @start_time = Time.now
+      @previous_time = @start_time
+      @title_width = 24
+      @format = "%-#{@title_width}s %3d%% %s %s"
+      @format_arguments = [:title, :percentage, :bar, :stat]
+      clear
+      show
+    end
+    attr_reader   :title
+    attr_reader   :current
+    attr_reader   :total
+    attr_accessor :start_time
+
+    private
+    def fmt_bar
+      bar_width = do_percentage * @terminal_width / 100
+      sprintf("[%s%s]",
+              @bar_mark * bar_width,
+              " " *  (@terminal_width - bar_width))
+    end
 
-  def fmt_percentage
-    do_percentage
-  end
+    def fmt_percentage
+      do_percentage
+    end
 
-  def fmt_stat
-    if @finished_p then elapsed else eta end
-  end
+    def fmt_stat
+      if @finished_p then elapsed else eta end
+    end
 
-  def fmt_stat_for_file_transfer
-    if @finished_p then 
-      sprintf("%s %s %s", bytes, transfer_rate, elapsed)
-    else 
-      sprintf("%s %s %s", bytes, transfer_rate, eta)
+    def fmt_stat_for_file_transfer
+      if @finished_p then
+        sprintf("%s %s %s", bytes, transfer_rate, elapsed)
+      else
+        sprintf("%s %s %s", bytes, transfer_rate, eta)
+      end
     end
-  end
 
-  def fmt_title
-    @title[0,(@title_width - 1)] + ":"
-  end
+    def fmt_title
+      @title[0,(@title_width - 1)] + ":"
+    end
+
+    def convert_bytes (bytes)
+      if bytes < 1024
+        sprintf("%6dB", bytes)
+      elsif bytes < 1024 * 1000 # 1000kb
+        sprintf("%5.1fKB", bytes.to_f / 1024)
+      elsif bytes < 1024 * 1024 * 1000  # 1000mb
+        sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
+      else
+        sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+      end
+    end
 
-  def convert_bytes (bytes)
-    if bytes < 1024
-      sprintf("%6dB", bytes)
-    elsif bytes < 1024 * 1000 # 1000kb
-      sprintf("%5.1fKB", bytes.to_f / 1024)
-    elsif bytes < 1024 * 1024 * 1000  # 1000mb
-      sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
-    else
-      sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+    def transfer_rate
+      bytes_per_second = @current.to_f / (Time.now - @start_time)
+      sprintf("%s/s", convert_bytes(bytes_per_second))
     end
-  end
 
-  def transfer_rate
-    bytes_per_second = @current.to_f / (Time.now - @start_time)
-    sprintf("%s/s", convert_bytes(bytes_per_second))
-  end
+    def bytes
+      convert_bytes(@current)
+    end
 
-  def bytes
-    convert_bytes(@current)
-  end
+    def format_time (t)
+      t = t.to_i
+      sec = t % 60
+      min  = (t / 60) % 60
+      hour = t / 3600
+      sprintf("%02d:%02d:%02d", hour, min, sec);
+    end
 
-  def format_time (t)
-    t = t.to_i
-    sec = t % 60
-    min  = (t / 60) % 60
-    hour = t / 3600
-    sprintf("%02d:%02d:%02d", hour, min, sec);
-  end
+    # ETA stands for Estimated Time of Arrival.
+    def eta
+      if @current == 0
+        "ETA:  --:--:--"
+      else
+        elapsed = Time.now - @start_time
+        eta = elapsed * @total / @current - elapsed;
+        sprintf("ETA:  %s", format_time(eta))
+      end
+    end
 
-  # ETA stands for Estimated Time of Arrival.
-  def eta
-    if @current == 0
-      "ETA:  --:--:--"
-    else
+    def elapsed
       elapsed = Time.now - @start_time
-      eta = elapsed * @total / @current - elapsed;
-      sprintf("ETA:  %s", format_time(eta))
+      sprintf("Time: %s", format_time(elapsed))
     end
-  end
-
-  def elapsed
-    elapsed = Time.now - @start_time
-    sprintf("Time: %s", format_time(elapsed))
-  end
-  
-  def eol
-    if @finished_p then "\n" else "\r" end
-  end
 
-  def do_percentage
-    if @total.zero?
-      100
-    else
-      @current  * 100 / @total
+    def eol
+      if @finished_p then "\n" else "\r" end
     end
-  end
 
-  def get_width
-    # FIXME: I don't know how portable it is.
-    default_width = 80
-    begin
-      tiocgwinsz = 0x5413
-      data = [0, 0, 0, 0].pack("SSSS")
-      if @out.ioctl(tiocgwinsz, data) >= 0 then
-        rows, cols, xpixels, ypixels = data.unpack("SSSS")
-        if cols >= 0 then cols else default_width end
+    def do_percentage
+      if @total.zero?
+        100
       else
-        default_width
+        @current  * 100 / @total
       end
-    rescue Exception
-      default_width
     end
-  end
 
-  def show
-    arguments = @format_arguments.map {|method| 
-      method = sprintf("fmt_%s", method)
-      send(method)
-    }
-    line = sprintf(@format, *arguments)
-
-    width = get_width
-    if line.length == width - 1 
-      @out.print(line + eol)
-      @out.flush
-    elsif line.length >= width
-      @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
-      if @terminal_width == 0 then @out.print(line + eol) else show end
-    else # line.length < width - 1
-      @terminal_width += width - line.length + 1
-      show
+    def show
+      arguments = @format_arguments.map {|method|
+        method = sprintf("fmt_%s", method)
+        send(method)
+      }
+      line = sprintf(@format, *arguments)
+
+      width = terminal_width(80)
+      if line.length == width - 1
+        @out.print(line + eol)
+        @out.flush
+      elsif line.length >= width
+        @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+        if @terminal_width == 0 then @out.print(line + eol) else show end
+      else # line.length < width - 1
+        @terminal_width += width - line.length + 1
+        show
+      end
+      @previous_time = Time.now
     end
-    @previous_time = Time.now
-  end
 
-  def show_if_needed
-    if @total.zero?
-      cur_percentage = 100
-      prev_percentage = 0
-    else
-      cur_percentage  = (@current  * 100 / @total).to_i
-      prev_percentage = (@previous * 100 / @total).to_i
-    end
+    def show_if_needed
+      if @total.zero?
+        cur_percentage = 100
+        prev_percentage = 0
+      else
+        cur_percentage  = (@current  * 100 / @total).to_i
+        prev_percentage = (@previous * 100 / @total).to_i
+      end
 
-    # Use "!=" instead of ">" to support negative changes
-    if cur_percentage != prev_percentage || 
-        Time.now - @previous_time >= 1 || @finished_p
-      show
+      # Use "!=" instead of ">" to support negative changes
+      if cur_percentage != prev_percentage ||
+          Time.now - @previous_time >= 1 || @finished_p
+        show
+      end
     end
-  end
 
-  public
-  def clear
-    @out.print "\r"
-    @out.print(" " * (get_width - 1))
-    @out.print "\r"
-  end
+    public
+    def clear
+      @out.print "\r"
+      @out.print(" " * (terminal_width(80) - 1))
+      @out.print "\r"
+    end
 
-  def finish
-    @current = @total
-    @finished_p = true
-    show
-  end
+    def finish
+      @current = @total
+      @finished_p = true
+      show
+    end
 
-  def finished?
-    @finished_p
-  end
+    def finished?
+      @finished_p
+    end
 
-  def file_transfer_mode
-    @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
-  end
+    def file_transfer_mode
+      @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
+    end
 
-  def format= (format)
-    @format = format
-  end
+    def format= (format)
+      @format = format
+    end
 
-  def format_arguments= (arguments)
-    @format_arguments = arguments
-  end
+    def format_arguments= (arguments)
+      @format_arguments = arguments
+    end
 
-  def halt
-    @finished_p = true
-    show
-  end
+    def halt
+      @finished_p = true
+      show
+    end
 
-  def inc (step = 1)
-    @current += step
-    @current = @total if @current > @total
-    show_if_needed
-    @previous = @current
-  end
+    def inc (step = 1)
+      @current += step
+      @current = @total if @current > @total
+      show_if_needed
+      @previous = @current
+    end
 
-  def set (count)
-    count = 0 if count < 0 
-    count = @total if count > @total
-    
-    @current = count
-    show_if_needed
-    @previous = @current
-  end
+    def set (count)
+      count = 0 if count < 0
+      count = @total if count > @total
 
-  def inspect
-    "#<ProgressBar:#{@current}/#{@total}>"
-  end
-end
+      @current = count
+      show_if_needed
+      @previous = @current
+    end
 
-class ReversedProgressBar < ProgressBar
-  def do_percentage
-    100 - super
+    def inspect
+      "#<ProgressBar:#{@current}/#{@total}>"
+    end
   end
 end
-

data/lib/cli/tools.rb

@@ -0,0 +1,49 @@
+# Try to determine the terminal with.
+# If it is not possible to to so, it returns the default_width.
+# <tt>default_width</tt> Defaults to 81
+def terminal_width(default_width = 81)
+  
+  begin
+    tiocgwinsz = 0x5413
+    data = [0, 0, 0, 0].pack("SSSS")
+    if !RUBY_PLATFORM.include?('java') && @out.ioctl(tiocgwinsz, data) >= 0 # JRuby crashes on ioctl
+      rows, cols, xpixels, ypixels = data.unpack("SSSS")
+      raise unless cols > 0
+      cols
+    else
+      raise
+    end
+  rescue 
+    begin
+      IO.popen('stty -a 2>&1') do |pipe|
+        column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
+        raise unless column_line
+        $1.to_i
+      end
+    rescue
+      default_width
+    end
+  end
+end
+
+# Copies request-log-analyzer analyzer rake tasks into the /lib/tasks folder of a project, for easy access and
+# environment integration.
+# <tt>install_type</tt> Type of project to install into. Defaults to :rails.
+# Raises if it cannot find the project folder or if the install_type is now known.
+def install_rake_tasks(install_type = :rails)
+  if install_type.to_sym == :rails
+    require 'ftools'
+    if File.directory?('./lib/tasks/')
+      File.copy(File.dirname(__FILE__) + '/../../tasks/request_log_analyzer.rake', './lib/tasks/request_log_analyze.rake')
+      puts "Installed rake tasks."
+      puts "To use, run: rake rla:report"
+    else
+      puts "Cannot find /lib/tasks folder. Are you in your Rails directory?"
+      puts "Installation aborted."
+    end
+  else
+    raise "Cannot perform this install type! (#{install_type.to_s})"
+  end
+end
+
+

data/lib/request_log_analyzer/aggregator/database_inserter.rb

@@ -0,0 +1,83 @@
+
+module RequestLogAnalyzer::Aggregator
+
+  # The database aggregator will create an SQLite3 database with all parsed request information.
+  #
+  # The prepare method will create a database schema according to the file format definitions.
+  # It will also create ActiveRecord::Base subclasses to interact with the created tables.
+  # Then, the aggregate method will be called for every parsed request. The information of
+  # these requests is inserted into the tables using the ActiveRecord classes.
+  #
+  # A requests table will be created, in which a record is inserted for every parsed request.
+  # For every line type, a separate table will be created with a request_id field to point to
+  # the request record, and a field for every parsed value. Finally, a warnings table will be
+  # created to log all parse warnings.
+  class DatabaseInserter < Base
+
+    attr_reader :request_count, :sources, :database
+
+    # Establishes a connection to the database and creates the necessary database schema for the
+    # current file format
+    def prepare
+      @sources = {}
+      @database = RequestLogAnalyzer::Database.new(options[:database])
+      @database.file_format = source.file_format
+
+      database.drop_database_schema! if options[:reset_database]
+      database.create_database_schema!
+    end
+
+    # Aggregates a request into the database
+    # This will create a record in the requests table and create a record for every line that has been parsed,
+    # in which the captured values will be stored.
+    def aggregate(request)
+      @request_object = RequestLogAnalyzer::Database::Request.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
+      request.lines.each do |line|
+        class_columns = database.get_class(line[:line_type]).column_names.reject { |column| ['id', 'source_id', 'request_id'].include?(column) }
+        attributes = Hash[*line.select { |(k, v)| class_columns.include?(k.to_s) }.flatten]
+        attributes[:source_id] = @sources[line[:source]].id if @sources[line[:source]]
+        @request_object.send("#{line[:line_type]}_lines").build(attributes)
+      end
+      @request_object.save!
+    rescue SQLite3::SQLException => e
+      raise Interrupt, e.message
+    end
+
+    # Finalizes the aggregator by closing the connection to the database
+    def finalize
+      @request_count = RequestLogAnalyzer::Database::Request.count
+      database.disconnect
+      database.remove_orm_classes!
+    end
+
+    # Records w warining in the warnings table.
+    def warning(type, message, lineno)
+      RequestLogAnalyzer::Database::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
+    end
+
+    # Records source changes in the sources table
+    def source_change(change, filename)
+      if File.exist?(filename)
+        case change
+        when :started
+          @sources[filename] = RequestLogAnalyzer::Database::Source.create!(:filename => filename)
+        when :finished
+          @sources[filename].update_attributes!(:filesize => File.size(filename), :mtime => File.mtime(filename))
+        end
+      end
+    end
+
+    # Prints a short report of what has been inserted into the database
+    def report(output)
+      output.title('Request database created')
+
+      output <<  "A database file has been created with all parsed request information.\n"
+      output <<  "#{@request_count} requests have been added to the database.\n"
+      output << "\n"
+      output <<  "To open a Ruby console to inspect the database, run the following command.\n"
+      output <<  output.colorize("  $ request-log-analyzer console -d #{options[:database]}\n", :bold)
+      output << "\n"
+    end
+
+  end
+end

data/lib/request_log_analyzer/aggregator/echo.rb

@@ -1,24 +1,29 @@
 module RequestLogAnalyzer::Aggregator
 
+  # Echo Aggregator. Writes everything to the screen when it is passed to this aggregator
   class Echo < Base
-    
+
+    attr_accessor :warnings
+
     def prepare
-      @warnings = ""
+      @warnings = []
     end
-    
+
+    # Display every parsed line immediately to the terminal
     def aggregate(request)
-      puts "\nRequest: " + request.inspect
+      puts "\nRequest: \n" + request.lines.map { |l| 
+        "\t#{l[:lineno]}:#{l[:line_type]}: #{l.reject { |(k,v)| [:lineno, :line_type].include?(k) }.inspect}" }.join("\n")
     end
-    
+
+    # Capture all warnings during parsing
     def warning(type, message, lineno)
-      @warnings << "WARNING #{type.inspect} on line #{lineno}: #{message}\n"
+      @warnings << "WARNING #{type.inspect} on line #{lineno}: #{message}"
     end
-    
-    def report(output=STDOUT, report_width = 80, color = false)
-      output << "\n"
-      output << "Warnings during parsing:\n"
-      output << green("━" * report_width, color) + "\n"
-      output << @warnings + "\n"
+
+    # Display every warning in the report when finished parsing
+    def report(output)
+      output.title("Warnings during parsing")
+      @warnings.each { |w| output.puts(w) }
     end
 
   end