request-log-analyzer 1.0.2

data/lib/cli/progressbar.rb

@@ -0,0 +1,236 @@
+#
+# Ruby/ProgressBar - a text progress bar library
+#
+# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms
+# of Ruby's license.
+#
+
+class ProgressBar
+  VERSION = "0.9"
+
+  def initialize (title, total, out = STDERR)
+    @title = title
+    @total = total
+    @out = out
+    @terminal_width = 80
+    @bar_mark = '='
+    @current = 0
+    @previous = 0
+    @finished_p = false
+    @start_time = Time.now
+    @previous_time = @start_time
+    @title_width = 24
+    @format = "%-#{@title_width}s %3d%% %s %s"
+    @format_arguments = [:title, :percentage, :bar, :stat]
+    clear
+    show
+  end
+  attr_reader   :title
+  attr_reader   :current
+  attr_reader   :total
+  attr_accessor :start_time
+
+  private
+  def fmt_bar
+    bar_width = do_percentage * @terminal_width / 100
+    sprintf("[%s%s]", 
+            @bar_mark * bar_width, 
+            " " *  (@terminal_width - bar_width))
+  end
+
+  def fmt_percentage
+    do_percentage
+  end
+
+  def fmt_stat
+    if @finished_p then elapsed else eta end
+  end
+
+  def fmt_stat_for_file_transfer
+    if @finished_p then 
+      sprintf("%s %s %s", bytes, transfer_rate, elapsed)
+    else 
+      sprintf("%s %s %s", bytes, transfer_rate, eta)
+    end
+  end
+
+  def fmt_title
+    @title[0,(@title_width - 1)] + ":"
+  end
+
+  def convert_bytes (bytes)
+    if bytes < 1024
+      sprintf("%6dB", bytes)
+    elsif bytes < 1024 * 1000 # 1000kb
+      sprintf("%5.1fKB", bytes.to_f / 1024)
+    elsif bytes < 1024 * 1024 * 1000  # 1000mb
+      sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
+    else
+      sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+    end
+  end
+
+  def transfer_rate
+    bytes_per_second = @current.to_f / (Time.now - @start_time)
+    sprintf("%s/s", convert_bytes(bytes_per_second))
+  end
+
+  def bytes
+    convert_bytes(@current)
+  end
+
+  def format_time (t)
+    t = t.to_i
+    sec = t % 60
+    min  = (t / 60) % 60
+    hour = t / 3600
+    sprintf("%02d:%02d:%02d", hour, min, sec);
+  end
+
+  # ETA stands for Estimated Time of Arrival.
+  def eta
+    if @current == 0
+      "ETA:  --:--:--"
+    else
+      elapsed = Time.now - @start_time
+      eta = elapsed * @total / @current - elapsed;
+      sprintf("ETA:  %s", format_time(eta))
+    end
+  end
+
+  def elapsed
+    elapsed = Time.now - @start_time
+    sprintf("Time: %s", format_time(elapsed))
+  end
+  
+  def eol
+    if @finished_p then "\n" else "\r" end
+  end
+
+  def do_percentage
+    if @total.zero?
+      100
+    else
+      @current  * 100 / @total
+    end
+  end
+
+  def get_width
+    # FIXME: I don't know how portable it is.
+    default_width = 80
+    begin
+      tiocgwinsz = 0x5413
+      data = [0, 0, 0, 0].pack("SSSS")
+      if @out.ioctl(tiocgwinsz, data) >= 0 then
+        rows, cols, xpixels, ypixels = data.unpack("SSSS")
+        if cols >= 0 then cols else default_width end
+      else
+        default_width
+      end
+    rescue Exception
+      default_width
+    end
+  end
+
+  def show
+    arguments = @format_arguments.map {|method| 
+      method = sprintf("fmt_%s", method)
+      send(method)
+    }
+    line = sprintf(@format, *arguments)
+
+    width = get_width
+    if line.length == width - 1 
+      @out.print(line + eol)
+      @out.flush
+    elsif line.length >= width
+      @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+      if @terminal_width == 0 then @out.print(line + eol) else show end
+    else # line.length < width - 1
+      @terminal_width += width - line.length + 1
+      show
+    end
+    @previous_time = Time.now
+  end
+
+  def show_if_needed
+    if @total.zero?
+      cur_percentage = 100
+      prev_percentage = 0
+    else
+      cur_percentage  = (@current  * 100 / @total).to_i
+      prev_percentage = (@previous * 100 / @total).to_i
+    end
+
+    # Use "!=" instead of ">" to support negative changes
+    if cur_percentage != prev_percentage || 
+        Time.now - @previous_time >= 1 || @finished_p
+      show
+    end
+  end
+
+  public
+  def clear
+    @out.print "\r"
+    @out.print(" " * (get_width - 1))
+    @out.print "\r"
+  end
+
+  def finish
+    @current = @total
+    @finished_p = true
+    show
+  end
+
+  def finished?
+    @finished_p
+  end
+
+  def file_transfer_mode
+    @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
+  end
+
+  def format= (format)
+    @format = format
+  end
+
+  def format_arguments= (arguments)
+    @format_arguments = arguments
+  end
+
+  def halt
+    @finished_p = true
+    show
+  end
+
+  def inc (step = 1)
+    @current += step
+    @current = @total if @current > @total
+    show_if_needed
+    @previous = @current
+  end
+
+  def set (count)
+    count = 0 if count < 0 
+    count = @total if count > @total
+    
+    @current = count
+    show_if_needed
+    @previous = @current
+  end
+
+  def inspect
+    "#<ProgressBar:#{@current}/#{@total}>"
+  end
+end
+
+class ReversedProgressBar < ProgressBar
+  def do_percentage
+    100 - super
+  end
+end
+

data/lib/request_log_analyzer.rb

@@ -0,0 +1,14 @@
+require 'date'
+require File.dirname(__FILE__) + '/cli/progressbar'
+require File.dirname(__FILE__) + '/cli/bashcolorizer'
+
+require File.dirname(__FILE__) + '/request_log_analyzer/file_format'
+require File.dirname(__FILE__) + '/request_log_analyzer/line_definition'
+require File.dirname(__FILE__) + '/request_log_analyzer/request'
+require File.dirname(__FILE__) + '/request_log_analyzer/log_parser'
+require File.dirname(__FILE__) + '/request_log_analyzer/aggregator/base'
+require File.dirname(__FILE__) + '/request_log_analyzer/aggregator/summarizer'
+require File.dirname(__FILE__) + '/request_log_analyzer/filter/base'
+require File.dirname(__FILE__) + '/request_log_analyzer/controller'
+require File.dirname(__FILE__) + '/request_log_analyzer/source/base'
+require File.dirname(__FILE__) + '/request_log_analyzer/source/log_file'

data/lib/request_log_analyzer/aggregator/base.rb

@@ -0,0 +1,45 @@
+module RequestLogAnalyzer::Aggregator
+
+  # The base class of an aggregator. This class provides the interface to which
+  # every aggregator should comply (by simply subclassing this class).
+  class Base
+    
+    include RequestLogAnalyzer::FileFormat::Awareness
+    
+    attr_reader :options
+    attr_reader :source
+
+    # Intializes a new RequestLogAnalyzer::Aggregator::Base instance
+    # It will include the specific file format module.
+    def initialize(source, options = {})
+      @source = source
+      self.register_file_format(source.file_format)
+      @options = options
+    end
+
+    # The prepare function is called just before parsing starts. This function 
+    # can be used to initialie variables, etc.
+    def prepare
+    end
+    
+    # The aggregate function is called for every request.
+    # Implement the aggregating functionality in this method
+    def aggregate(request)
+    end
+    
+    # The finalize function is called after all sources are parsed and no more
+    # requests will be passed to the aggregator
+    def finalize
+    end
+    
+    # The warning method is called if the parser eits a warning.
+    def warning(type, message, lineno)
+    end    
+    
+    # The report function is called at the end. Implement any result reporting
+    # in this function.
+    def report(output = STDOUT, report_width = 80, color = false)
+    end
+
+  end
+end

data/lib/request_log_analyzer/aggregator/database.rb

@@ -0,0 +1,148 @@
+require 'rubygems'
+require 'activerecord'
+
+module RequestLogAnalyzer::Aggregator
+
+  # The database aggregator will create an SQLite3 database with all parsed request information.
+  #
+  # The prepare method will create a database schema according to the file format definitions.
+  # It will also create ActiveRecord::Base subclasses to interact with the created tables. 
+  # Then, the aggregate method will be called for every parsed request. The information of
+  # these requests is inserted into the tables using the ActiveRecord classes.
+  #
+  # A requests table will be created, in which a record is inserted for every parsed request.
+  # For every line type, a separate table will be created with a request_id field to point to
+  # the request record, and a field for every parsed value. Finally, a warnings table will be
+  # created to log all parse warnings.
+  class Database < Base
+
+    # Establishes a connection to the database and creates the necessary database schema for the
+    # current file format
+    def prepare
+      ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => options[:database])
+      File.unlink(options[:database]) if File.exist?(options[:database]) # TODO: keep old database?
+      create_database_schema!
+    end
+    
+    # Aggregates a request into the database
+    # This will create a record in the requests table and create a record for every line that has been parsed,
+    # in which the captured values will be stored.
+    def aggregate(request)
+      @request_object = @request_class.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
+      request.lines.each do |line|
+        attributes = line.reject { |k, v| [:line_type].include?(k) }
+        @request_object.send("#{line[:line_type]}_lines").build(attributes)
+      end
+      @request_object.save!
+    rescue SQLite3::SQLException => e
+      raise Interrupt, e.message
+    end
+    
+    # Finalizes the aggregator by closing the connection to the database
+    def finalize
+      @request_count = @orm_module::Request.count
+      ActiveRecord::Base.remove_connection
+    end
+    
+    # Records w warining in the warnings table.
+    def warning(type, message, lineno)
+      @orm_module::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
+    end
+    
+    # Prints a short report of what has been inserted into the database
+    def report(output = STDOUT, report_width = 80, color = false)
+      output << "\n"
+      output << green("━" * report_width, color) + "\n"
+      output <<  "A database file has been created with all parsed request information.\n"
+      output <<  "#{@request_count} requests have been added to the database.\n"      
+      output <<  "To execute queries on this database, run the following command:\n"
+      output <<  "  $ sqlite3 #{options[:database]}\n"
+      output << "\n"
+    end
+    
+    protected 
+    
+    # This function creates a database table for a given line definition.
+    # It will create a field for every capture in the line, and adds a lineno field to indicate at
+    # what line in the original file the line was found, and a request_id to link lines related
+    # to the same request. It will also create an index in the request_id field to speed up queries.
+    def create_database_table(name, definition)
+      ActiveRecord::Migration.verbose = options[:debug]
+      ActiveRecord::Migration.create_table("#{name}_lines") do |t|
+        t.column(:request_id, :integer)
+        t.column(:lineno, :integer)
+        definition.captures.each do |capture|
+          t.column(capture[:name], column_type(capture))
+        end
+      end
+      ActiveRecord::Migration.add_index("#{name}_lines", [:request_id])
+    end
+    
+    # Creates an ActiveRecord class for a given line definition.
+    # A subclass of ActiveRecord::Base is created and an association with the Request class is
+    # created using belongs_to / has_many. This association will later be used to create records
+    # in the corresponding table. This table should already be created before this method is called.
+    def create_activerecord_class(name, definition)
+      class_name = "#{name}_line".camelize
+      klass = Class.new(ActiveRecord::Base)
+      klass.send(:belongs_to, :request)
+      @orm_module.const_set(class_name, klass) unless @orm_module.const_defined?(class_name)
+      @request_class.send(:has_many, "#{name}_lines".to_sym)
+    end    
+    
+    # Creates a requests table, in which a record is created for every request. It also creates an
+    # ActiveRecord::Base class to communicate with this table.
+    def create_request_table_and_class
+      ActiveRecord::Migration.verbose = options[:debug]
+      ActiveRecord::Migration.create_table("requests") do |t|
+        t.integer :first_lineno
+        t.integer :last_lineno
+      end    
+      
+      @orm_module.const_set('Request', Class.new(ActiveRecord::Base)) unless @orm_module.const_defined?('Request')     
+      @request_class = @orm_module.const_get('Request')
+    end
+
+    # Creates a warnings table and a corresponding Warning class to communicate with this table using ActiveRecord.
+    def create_warning_table_and_class
+      ActiveRecord::Migration.verbose = options[:debug]
+      ActiveRecord::Migration.create_table("warnings") do |t|
+        t.string  :warning_type, :limit => 30, :null => false
+        t.string  :message
+        t.integer :lineno          
+      end    
+      
+      @orm_module.const_set('Warning', Class.new(ActiveRecord::Base)) unless @orm_module.const_defined?('Warning')
+    end
+    
+    # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the 
+    # file format definition. These ORM classes will later be used to create records in the database.
+    def create_database_schema!
+      
+      if file_format.class.const_defined?('Database')
+        @orm_module = file_format.class.const_get('Database')
+      else
+        @orm_module = file_format.class.const_set('Database', Module.new)
+      end
+
+      create_request_table_and_class
+      create_warning_table_and_class
+      
+      file_format.line_definitions.each do |name, definition|
+        create_database_table(name, definition)
+        create_activerecord_class(name, definition)
+      end
+    end
+    
+    # Function to determine the column type for a field
+    # TODO: make more robust / include in file-format definition
+    def column_type(capture)
+      case capture[:type]
+      when :sec;   :double
+      when :msec;  :double
+      when :float; :double
+      else         capture[:type]
+      end
+    end
+  end
+end

data/lib/request_log_analyzer/aggregator/echo.rb

@@ -0,0 +1,25 @@
+module RequestLogAnalyzer::Aggregator
+
+  class Echo < Base
+    
+    def prepare
+      @warnings = ""
+    end
+    
+    def aggregate(request)
+      puts "\nRequest: " + request.inspect
+    end
+    
+    def warning(type, message, lineno)
+      @warnings << "WARNING #{type.inspect} on line #{lineno}: #{message}\n"
+    end
+    
+    def report(output=STDOUT, report_width = 80, color = false)
+      output << "\n"
+      output << "Warnings during parsing:\n"
+      output << green("━" * report_width, color) + "\n"
+      output << @warnings + "\n"
+    end
+
+  end
+end