ngmoco-request-log-analyzer 1.4.2

data/lib/cli/database_console.rb

@@ -0,0 +1,26 @@
+
+class DatabaseConsole
+
+  IRB = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+  def initialize(arguments)
+    @arguments = arguments
+  end
+
+  def run!
+    libraries = ['irb/completion', 'rubygems', './lib/request_log_analyzer', './lib/cli/database_console_init']
+    libaries_string = libraries.map { |l| "-r #{l}" }.join(' ')
+
+    ENV['RLA_DBCONSOLE_DATABASE'] = @arguments[:database]
+    if @arguments[:apache_format]
+      ENV['RLA_DBCONSOLE_FORMAT'] = 'apache'
+      ENV['RLA_DBCONSOLE_FORMAT_ARGUMENT'] = @arguments[:apache_format]
+    else
+      ENV['RLA_DBCONSOLE_FORMAT'] = @arguments[:format]
+    end
+    # ENV['RLA_DBCONSOLE_FORMAT_ARGS'] = arguments['database']
+
+    exec("#{IRB} #{libaries_string} --simple-prompt")
+  end
+end
+

data/lib/cli/database_console_init.rb

@@ -0,0 +1,43 @@
+# Setup the include path
+$:.unshift(File.dirname(__FILE__) + '/..')
+
+$database = RequestLogAnalyzer::Database.new(ENV['RLA_DBCONSOLE_DATABASE'])
+$database.load_database_schema!
+$database.register_default_orm_classes!
+
+require 'cli/tools'
+
+def wordwrap(string, max = 80, indent = "")
+  strings = [""]
+  string.split(", ").each do |item|
+    if strings.last.length == 0 || strings.last.length + item.length <= max
+      strings.last << item << ', '
+    else
+      strings << (item + ', ')
+    end
+  end
+  strings.map(&:strip).join("\n#{indent}").slice(0..-2)
+end
+
+class Request
+  def inspect
+    request_inspect = "Request[id: #{id}]"
+    request_inspect << " <#{lines.first.source.filename}>" if lines.first.source
+
+    inspected_lines = lines.map do |line|
+      inspect_line = "   - #{line.line_type} (line #{line.lineno})"
+      if (inspect_attributes = line.attributes.reject { |(k, v)| [:id, :source_id, :request_id, :lineno].include?(k.to_sym) }).any?
+        inspect_attributes = inspect_attributes.map { |(k,v)| "#{k} = #{v.inspect}" }.join(', ')
+        inspect_line << "\n      " + wordwrap(inspect_attributes, terminal_width - 6, "      ")
+      end
+      inspect_line
+    end
+
+    request_inspect << "\n" << inspected_lines.join("\n") << "\n\n"
+  end
+end
+
+puts "request-log-analyzer database console"
+puts "-------------------------------------"
+puts "The following ActiveRecord classes are available:"
+puts $database.orm_classes.map { |k| k.name.split('::').last }.join(", ")

data/lib/cli/progressbar.rb

@@ -0,0 +1,213 @@
+#
+# Ruby/ProgressBar - a text progress bar library
+#
+# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms
+# of Ruby's license.
+
+module CommandLine
+  class ProgressBar
+    VERSION = "0.9"
+
+    def initialize (title, total, out = STDERR)
+      @title = title
+      @total = total
+      @out = out
+      @terminal_width = 80
+      @bar_mark = '='
+      @current = 0
+      @previous = 0
+      @finished_p = false
+      @start_time = Time.now
+      @previous_time = @start_time
+      @title_width = 24
+      @format = "%-#{@title_width}s %3d%% %s %s"
+      @format_arguments = [:title, :percentage, :bar, :stat]
+      clear
+      show
+    end
+    attr_reader   :title
+    attr_reader   :current
+    attr_reader   :total
+    attr_accessor :start_time
+
+    private
+    def fmt_bar
+      bar_width = do_percentage * @terminal_width / 100
+      sprintf("[%s%s]",
+              @bar_mark * bar_width,
+              " " *  (@terminal_width - bar_width))
+    end
+
+    def fmt_percentage
+      do_percentage
+    end
+
+    def fmt_stat
+      if @finished_p then elapsed else eta end
+    end
+
+    def fmt_stat_for_file_transfer
+      if @finished_p then
+        sprintf("%s %s %s", bytes, transfer_rate, elapsed)
+      else
+        sprintf("%s %s %s", bytes, transfer_rate, eta)
+      end
+    end
+
+    def fmt_title
+      @title[0,(@title_width - 1)] + ":"
+    end
+
+    def convert_bytes (bytes)
+      if bytes < 1024
+        sprintf("%6dB", bytes)
+      elsif bytes < 1024 * 1000 # 1000kb
+        sprintf("%5.1fKB", bytes.to_f / 1024)
+      elsif bytes < 1024 * 1024 * 1000  # 1000mb
+        sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
+      else
+        sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+      end
+    end
+
+    def transfer_rate
+      bytes_per_second = @current.to_f / (Time.now - @start_time)
+      sprintf("%s/s", convert_bytes(bytes_per_second))
+    end
+
+    def bytes
+      convert_bytes(@current)
+    end
+
+    def format_time (t)
+      t = t.to_i
+      sec = t % 60
+      min  = (t / 60) % 60
+      hour = t / 3600
+      sprintf("%02d:%02d:%02d", hour, min, sec);
+    end
+
+    # ETA stands for Estimated Time of Arrival.
+    def eta
+      if @current == 0
+        "ETA:  --:--:--"
+      else
+        elapsed = Time.now - @start_time
+        eta = elapsed * @total / @current - elapsed;
+        sprintf("ETA:  %s", format_time(eta))
+      end
+    end
+
+    def elapsed
+      elapsed = Time.now - @start_time
+      sprintf("Time: %s", format_time(elapsed))
+    end
+
+    def eol
+      if @finished_p then "\n" else "\r" end
+    end
+
+    def do_percentage
+      if @total.zero?
+        100
+      else
+        @current  * 100 / @total
+      end
+    end
+
+    def show
+      arguments = @format_arguments.map {|method|
+        method = sprintf("fmt_%s", method)
+        send(method)
+      }
+      line = sprintf(@format, *arguments)
+
+      width = terminal_width(80)
+      if line.length == width - 1
+        @out.print(line + eol)
+        @out.flush
+      elsif line.length >= width
+        @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+        if @terminal_width == 0 then @out.print(line + eol) else show end
+      else # line.length < width - 1
+        @terminal_width += width - line.length + 1
+        show
+      end
+      @previous_time = Time.now
+    end
+
+    def show_if_needed
+      if @total.zero?
+        cur_percentage = 100
+        prev_percentage = 0
+      else
+        cur_percentage  = (@current  * 100 / @total).to_i
+        prev_percentage = (@previous * 100 / @total).to_i
+      end
+
+      # Use "!=" instead of ">" to support negative changes
+      if cur_percentage != prev_percentage ||
+          Time.now - @previous_time >= 1 || @finished_p
+        show
+      end
+    end
+
+    public
+    def clear
+      @out.print "\r"
+      @out.print(" " * (terminal_width(80) - 1))
+      @out.print "\r"
+    end
+
+    def finish
+      @current = @total
+      @finished_p = true
+      show
+    end
+
+    def finished?
+      @finished_p
+    end
+
+    def file_transfer_mode
+      @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
+    end
+
+    def format= (format)
+      @format = format
+    end
+
+    def format_arguments= (arguments)
+      @format_arguments = arguments
+    end
+
+    def halt
+      @finished_p = true
+      show
+    end
+
+    def inc (step = 1)
+      @current += step
+      @current = @total if @current > @total
+      show_if_needed
+      @previous = @current
+    end
+
+    def set (count)
+      count = 0 if count < 0
+      count = @total if count > @total
+
+      @current = count
+      show_if_needed
+      @previous = @current
+    end
+
+    def inspect
+      "#<ProgressBar:#{@current}/#{@total}>"
+    end
+  end
+end

data/lib/cli/tools.rb

@@ -0,0 +1,46 @@
+# Try to determine the terminal with.
+# If it is not possible to to so, it returns the default_width.
+# <tt>default_width</tt> Defaults to 81
+def terminal_width(default_width = 81)
+  tiocgwinsz = 0x5413
+  data = [0, 0, 0, 0].pack("SSSS")
+  if @out.ioctl(tiocgwinsz, data) >= 0
+    rows, cols, xpixels, ypixels = data.unpack("SSSS")
+    raise unless cols > 0
+    cols
+  else
+    raise
+  end
+rescue
+  begin
+    IO.popen('stty -a 2>&1') do |pipe|
+      column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
+      raise unless column_line
+      $1.to_i
+    end
+  rescue
+    default_width
+  end
+end
+
+# Copies request-log-analyzer analyzer rake tasks into the /lib/tasks folder of a project, for easy access and
+# environment integration.
+# <tt>install_type</tt> Type of project to install into. Defaults to :rails.
+# Raises if it cannot find the project folder or if the install_type is now known.
+def install_rake_tasks(install_type = :rails)
+  if install_type.to_sym == :rails
+    require 'ftools'
+    if File.directory?('./lib/tasks/')
+      File.copy(File.dirname(__FILE__) + '/../../tasks/request_log_analyzer.rake', './lib/tasks/request_log_analyze.rake')
+      puts "Installed rake tasks."
+      puts "To use, run: rake rla:report"
+    else
+      puts "Cannot find /lib/tasks folder. Are you in your Rails directory?"
+      puts "Installation aborted."
+    end
+  else
+    raise "Cannot perform this install type! (#{install_type.to_s})"
+  end
+end
+
+

data/lib/request_log_analyzer.rb

@@ -0,0 +1,44 @@
+require 'date'
+
+# Satisfy ruby 1.9 sensitivity about encoding.
+Encoding.default_external = 'binary' if defined? Encoding and Encoding.respond_to? 'default_external='
+
+# RequestLogAnalyzer is the base namespace in which all functionality of RequestLogAnalyzer is implemented.
+#
+# - This module itselfs contains some functions to help with class and source file loading.
+# - The actual application resides in the RequestLogAnalyzer::Controller class.
+module RequestLogAnalyzer
+
+  # The current version of request-log-analyzer.
+  # Do not change the value by hand; it will be updated automatically by the gem release script.
+  VERSION = "1.4.1"
+
+  # Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
+  # <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.
+  def self.const_missing(const)
+    load_default_class_file(RequestLogAnalyzer, const)
+  end
+
+  # Loads constants that reside in the RequestLogAnalyzer tree using the constant name
+  # and its base constant to determine the filename.
+  # <tt>base</tt>:: The base constant to load the constant from. This should be Foo when the constant Foo::Bar is being loaded.
+  # <tt>const</tt>:: The constant to load from the base constant as a string or symbol. This should be 'Bar' or :Bar when the constant Foo::Bar is being loaded.
+  def self.load_default_class_file(base, const)
+    require "#{to_underscore("#{base.name}::#{const}")}"
+    base.const_get(const) if base.const_defined?(const)
+  end
+
+  # Convert a string/symbol in camelcase (RequestLogAnalyzer::Controller) to underscores (request_log_analyzer/controller)
+  # This function can be used to load the file (using require) in which the given constant is defined.
+  # <tt>str</tt>:: The string to convert in the following format: <tt>ModuleName::ClassName</tt>
+  def self.to_underscore(str)
+    str.to_s.gsub(/::/, '/').gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').gsub(/([a-z\d])([A-Z])/,'\1_\2').tr("-", "_").downcase
+  end
+
+  # Convert a string/symbol in underscores (<tt>request_log_analyzer/controller</tt>) to camelcase
+  # (<tt>RequestLogAnalyzer::Controller</tt>). This can be used to find the class that is defined in a given filename.
+  # <tt>str</tt>:: The string to convert in the following format: <tt>module_name/class_name</tt>
+  def self.to_camelcase(str)
+    str.to_s.gsub(/\/(.?)/) { "::" + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
+  end
+end

data/lib/request_log_analyzer/aggregator.rb

@@ -0,0 +1,49 @@
+module RequestLogAnalyzer::Aggregator
+
+  def self.const_missing(const)
+    RequestLogAnalyzer::load_default_class_file(self, const)
+  end
+
+  # The base class of an aggregator. This class provides the interface to which
+  # every aggregator should comply (by simply subclassing this class).
+  class Base
+
+    attr_reader :options, :source
+
+    # Intializes a new RequestLogAnalyzer::Aggregator::Base instance
+    # It will include the specific file format module.
+    def initialize(source, options = {})
+      @source = source
+      @options = options
+    end
+
+    # The prepare function is called just before parsing starts. This function
+    # can be used to initialie variables, etc.
+    def prepare
+    end
+
+    # The aggregate function is called for every request.
+    # Implement the aggregating functionality in this method
+    def aggregate(request)
+    end
+
+    # The finalize function is called after all sources are parsed and no more
+    # requests will be passed to the aggregator
+    def finalize
+    end
+
+    # The warning method is called if the parser eits a warning.
+    def warning(type, message, lineno)
+    end
+
+    # The report function is called at the end. Implement any result reporting
+    # in this function.
+    def report(output)
+    end
+
+    # The source_change function gets called when handling a source is started or finished.
+    def source_change(change, filename)
+    end
+
+  end
+end

data/lib/request_log_analyzer/aggregator/database_inserter.rb

@@ -0,0 +1,83 @@
+
+module RequestLogAnalyzer::Aggregator
+
+  # The database aggregator will create an SQLite3 database with all parsed request information.
+  #
+  # The prepare method will create a database schema according to the file format definitions.
+  # It will also create ActiveRecord::Base subclasses to interact with the created tables.
+  # Then, the aggregate method will be called for every parsed request. The information of
+  # these requests is inserted into the tables using the ActiveRecord classes.
+  #
+  # A requests table will be created, in which a record is inserted for every parsed request.
+  # For every line type, a separate table will be created with a request_id field to point to
+  # the request record, and a field for every parsed value. Finally, a warnings table will be
+  # created to log all parse warnings.
+  class DatabaseInserter < Base
+
+    attr_reader :request_count, :sources, :database
+
+    # Establishes a connection to the database and creates the necessary database schema for the
+    # current file format
+    def prepare
+      @sources = {}
+      @database = RequestLogAnalyzer::Database.new(options[:database])
+      @database.file_format = source.file_format
+
+      database.drop_database_schema! if options[:reset_database]
+      database.create_database_schema!
+    end
+
+    # Aggregates a request into the database
+    # This will create a record in the requests table and create a record for every line that has been parsed,
+    # in which the captured values will be stored.
+    def aggregate(request)
+      @request_object = RequestLogAnalyzer::Database::Request.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
+      request.lines.each do |line|
+        class_columns = database.get_class(line[:line_type]).column_names.reject { |column| ['id', 'source_id', 'request_id'].include?(column) }
+        attributes = Hash[*line.select { |(k, v)| class_columns.include?(k.to_s) }.flatten]
+        attributes[:source_id] = @sources[line[:source]].id if @sources[line[:source]]
+        @request_object.send("#{line[:line_type]}_lines").build(attributes)
+      end
+      @request_object.save!
+    rescue SQLite3::SQLException => e
+      raise Interrupt, e.message
+    end
+
+    # Finalizes the aggregator by closing the connection to the database
+    def finalize
+      @request_count = RequestLogAnalyzer::Database::Request.count
+      database.disconnect
+      database.remove_orm_classes!
+    end
+
+    # Records w warining in the warnings table.
+    def warning(type, message, lineno)
+      RequestLogAnalyzer::Database::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
+    end
+
+    # Records source changes in the sources table
+    def source_change(change, filename)
+      if File.exist?(filename)
+        case change
+        when :started
+          @sources[filename] = RequestLogAnalyzer::Database::Source.create!(:filename => filename)
+        when :finished
+          @sources[filename].update_attributes!(:filesize => File.size(filename), :mtime => File.mtime(filename))
+        end
+      end
+    end
+
+    # Prints a short report of what has been inserted into the database
+    def report(output)
+      output.title('Request database created')
+
+      output <<  "A database file has been created with all parsed request information.\n"
+      output <<  "#{@request_count} requests have been added to the database.\n"
+      output << "\n"
+      output <<  "To open a Ruby console to inspect the database, run the following command.\n"
+      output <<  output.colorize("  $ request-log-analyzer console -d #{options[:database]}\n", :bold)
+      output << "\n"
+    end
+
+  end
+end