request-log-analyzer 1.2.9 → 1.3.0

data/bin/request-log-analyzer

@@ -1,8 +1,9 @@
 #!/usr/bin/ruby
-require File.dirname(__FILE__) + '/../lib/request_log_analyzer'
-require File.dirname(__FILE__) + '/../lib/cli/command_line_arguments'
-require File.dirname(__FILE__) + '/../lib/cli/progressbar'
-require File.dirname(__FILE__) + '/../lib/cli/tools'
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+require 'request_log_analyzer'
+require 'cli/command_line_arguments'
+require 'lib/cli/progressbar'
+require 'lib/cli/tools'
 
 # Parse the arguments given via commandline
 begin
@@ -11,40 +12,49 @@ begin
     command_line.command(:install) do |install|
       install.parameters = 1
     end
-    
+
+    command_line.command(:console) do |cons|
+      cons.option(:database, :alias => :d, :required => true)
+      cons.option(:format, :alias => :f, :default => 'rails')
+      cons.option(:apache_format)
+    end
+
     command_line.command(:strip) do |strip|
       strip.minimum_parameters = 1
-      strip.option(:format, :alias => :f, :default => 'rails')      
+      strip.option(:format, :alias => :f, :default => 'rails')
       strip.option(:output, :alias => :o)
       strip.switch(:discard_teaser_lines, :t)
       strip.switch(:keep_junk_lines, :j) 
     end
-    
+
     command_line.option(:format, :alias => :f, :default => 'rails')
     command_line.option(:apache_format)
-    
+
     command_line.option(:file, :alias => :e)
+    command_line.option(:mail, :alias => :m)
     command_line.option(:parse_strategy, :default => 'assume-correct')
     command_line.option(:dump)
-          
+
     command_line.option(:aggregator, :alias => :a, :multiple => true)
-    command_line.option(:database, :alias => :d)    
+
+    command_line.option(:database, :alias => :d)
+    command_line.switch(:reset_database)
 
     # filtering options
     command_line.option(:select, :multiple => true, :parameters => 2)
     command_line.option(:reject, :multiple => true, :parameters => 2)
     command_line.option(:after)
-    command_line.option(:before)    
-    
+    command_line.option(:before)
+
     command_line.switch(:boring, :b)
-    command_line.option(:output, :alias => :o, :default => 'FixedWidth')    
+    command_line.option(:output, :alias => :o, :default => 'FixedWidth')
     command_line.option(:report_width, :default => terminal_width - 1)
-    
+
     command_line.switch(:debug)
-    
-    command_line.minimum_parameters = 1    
+
+    command_line.minimum_parameters = 1
   end
-  
+
 rescue CommandLine::Error => e
   puts "Request-log-analyzer, by Willem van Bergen and Bart ten Brinke - version #{RequestLogAnalyzer::VERSION}"  
   puts "Website: http://railsdoctors.com"
@@ -65,6 +75,7 @@ rescue CommandLine::Error => e
   puts "  --database <filename>, -d: Creates an SQLite3 database of all the parsed request information."
   puts "  --debug                    Print debug information while parsing."
   puts "  --file <filename>          Output to file."
+  puts "  --mail <emailaddress>      Send report to an email address."
   puts "  --output <format>          Output format. Supports 'HTML' and 'FixedWidth' (default)"  
   puts "  --dump <filename>          Dump the YAML formatted results in the given file"
   puts 
@@ -83,11 +94,14 @@ end
 case arguments.command
 when :install
   install_rake_tasks(arguments.parameters[0])
+when :console
+  require 'cli/database_console'
+  DatabaseConsole.new(arguments).run!
 when :strip
-  require File.dirname(__FILE__) + '/../lib/request_log_analyzer/log_processor'  
+  require File.dirname(__FILE__) + '/../lib/request_log_analyzer/log_processor'
   RequestLogAnalyzer::LogProcessor.build(:strip, arguments).run!
 else 
-  puts "Request-log-analyzer, by Willem van Bergen and Bart ten Brinke - version #{RequestLogAnalyzer::VERSION}"  
+  puts "Request-log-analyzer, by Willem van Bergen and Bart ten Brinke - version #{RequestLogAnalyzer::VERSION}"
   puts "Website: http://railsdoctors.com"
   puts
 

data/lib/cli/database_console.rb

@@ -0,0 +1,26 @@
+
+class DatabaseConsole
+  
+  IRB = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+  
+  def initialize(arguments)
+    @arguments = arguments
+  end
+  
+  def run!
+    libraries = ['irb/completion', 'rubygems', './lib/request_log_analyzer', './lib/cli/database_console_init']
+    libaries_string = libraries.map { |l| "-r #{l}" }.join(' ')
+
+    ENV['RLA_DBCONSOLE_DATABASE'] = @arguments[:database]
+    if @arguments[:apache_format]
+      ENV['RLA_DBCONSOLE_FORMAT'] = 'apache'
+      ENV['RLA_DBCONSOLE_FORMAT_ARGUMENT'] = @arguments[:apache_format]
+    else
+      ENV['RLA_DBCONSOLE_FORMAT'] = @arguments[:format]
+    end
+    # ENV['RLA_DBCONSOLE_FORMAT_ARGS'] = arguments['database']
+
+    exec("#{IRB} #{libaries_string} --simple-prompt")
+  end
+end
+

data/lib/cli/database_console_init.rb

@@ -0,0 +1,42 @@
+# Setup the include path
+$:.unshift(File.dirname(__FILE__) + '/..')
+
+$database = RequestLogAnalyzer::Database.new(ENV['RLA_DBCONSOLE_DATABASE'])
+$database.load_database_schema!
+
+require 'cli/tools'
+
+def wordwrap(string, max = 80, indent = "")
+  strings = [""]
+  string.split(", ").each do |item|
+    if strings.last.length == 0 || strings.last.length + item.length <= max
+      strings.last << item << ', '
+    else
+      strings << (item + ', ')
+    end
+  end
+  strings.map(&:strip).join("\n#{indent}").slice(0..-2)
+end
+
+class Request
+  def inspect
+    request_inspect = "Request[id: #{id}]"
+    request_inspect << " <#{lines.first.source.filename}>" if lines.first.source
+    
+    inspected_lines = lines.map do |line|
+      inspect_line = "   - #{line.line_type} (line #{line.lineno})"
+      if (inspect_attributes = line.attributes.reject { |(k, v)| [:id, :source_id, :request_id, :lineno].include?(k.to_sym) }).any?
+        inspect_attributes = inspect_attributes.map { |(k,v)| "#{k} = #{v.inspect}" }.join(', ')
+        inspect_line << "\n      " + wordwrap(inspect_attributes, terminal_width - 6, "      ")
+      end
+      inspect_line
+    end
+    
+    request_inspect << "\n" << inspected_lines.join("\n") << "\n\n"
+  end
+end
+
+puts "request-log-analyzer database console"
+puts "-------------------------------------"
+puts "The following ActiveRecord classes are available:"
+puts $database.orm_classes.join(", ")

data/lib/cli/tools.rb

@@ -13,7 +13,7 @@ def terminal_width(default_width = 81)
   end
 rescue   
   begin 
-    IO.popen('stty -a') do |pipe|
+    IO.popen('stty -a 2>&1') do |pipe|
       column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
       raise unless column_line
       $1.to_i

data/lib/request_log_analyzer/aggregator/database_inserter.rb

@@ -0,0 +1,81 @@
+
+module RequestLogAnalyzer::Aggregator
+
+  # The database aggregator will create an SQLite3 database with all parsed request information.
+  #
+  # The prepare method will create a database schema according to the file format definitions.
+  # It will also create ActiveRecord::Base subclasses to interact with the created tables. 
+  # Then, the aggregate method will be called for every parsed request. The information of
+  # these requests is inserted into the tables using the ActiveRecord classes.
+  #
+  # A requests table will be created, in which a record is inserted for every parsed request.
+  # For every line type, a separate table will be created with a request_id field to point to
+  # the request record, and a field for every parsed value. Finally, a warnings table will be
+  # created to log all parse warnings.
+  class DatabaseInserter < Base
+
+    attr_reader :request_count, :sources, :database
+
+    # Establishes a connection to the database and creates the necessary database schema for the
+    # current file format
+    def prepare
+      @sources = {}
+      @database = RequestLogAnalyzer::Database.new(options[:database])
+      @database.file_format = source.file_format
+      
+      database.drop_database_schema! if options[:reset_database]
+      database.create_database_schema!
+    end
+    
+    # Aggregates a request into the database
+    # This will create a record in the requests table and create a record for every line that has been parsed,
+    # in which the captured values will be stored.
+    def aggregate(request)
+      @request_object = database.request_class.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
+      request.lines.each do |line|
+        class_columns = database.get_class(line[:line_type]).column_names.reject { |column| ['id', 'source_id', 'request_id'].include?(column) }
+        attributes = Hash[*line.select { |(k, v)| class_columns.include?(k.to_s) }.flatten]
+        attributes[:source_id] = @sources[line[:source]].id if @sources[line[:source]]
+        @request_object.send("#{line[:line_type]}_lines").build(attributes)
+      end
+      @request_object.save!
+    rescue SQLite3::SQLException => e
+      raise Interrupt, e.message
+    end
+    
+    # Finalizes the aggregator by closing the connection to the database
+    def finalize
+      @request_count = database.request_class.count
+      database.disconnect
+      database.remove_orm_classes!
+    end
+    
+    # Records w warining in the warnings table.
+    def warning(type, message, lineno)
+      database.warning_class.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
+    end
+    
+    # Records source changes in the sources table
+    def source_change(change, filename)
+      case change
+      when :started
+        @sources[filename] = database.source_class.create!(:filename => filename)
+      when :finished
+        @sources[filename].update_attributes!(:filesize => File.size(filename), :mtime => File.mtime(filename))
+      end
+    end
+    
+    # Prints a short report of what has been inserted into the database
+    def report(output)
+      output.title('Request database created')
+      
+      output <<  "A database file has been created with all parsed request information.\n"
+      output <<  "#{@request_count} requests have been added to the database.\n"
+      output << "\n"
+      output <<  "To open a Ruby console to inspect the database, run the following command.\n"
+      output <<  output.colorize("  $ request-log-analyzer console -d #{options[:database]}\n", :bold)
+      output << "\n"
+    end
+    
+  end
+end

data/lib/request_log_analyzer/aggregator/summarizer.rb

@@ -130,9 +130,9 @@ module RequestLogAnalyzer::Aggregator
       output.with_style(:cell_separator => false) do 
         output.table({:width => 20}, {:font => :bold}) do |rows|
           rows << ['Parsed lines:',    source.parsed_lines]
-          rows << ['Parsed requests:', source.parsed_requests]
           rows << ['Skipped lines:',   source.skipped_lines]
-        
+          rows << ['Parsed requests:', source.parsed_requests]
+          rows << ['Skipped requests:', source.skipped_requests]
           rows << ["Warnings:", @warnings_encountered.map { |(key, value)| "#{key}: #{value}" }.join(', ')] if has_warnings?
         end
       end

data/lib/request_log_analyzer/aggregator.rb

@@ -44,6 +44,10 @@ module RequestLogAnalyzer::Aggregator
     # in this function.
     def report(output)
     end
+    
+    # The source_change function gets called when handling a source is started or finished.
+    def source_change(change, filename)
+    end
 
   end
 end

data/lib/request_log_analyzer/controller.rb

@@ -32,7 +32,10 @@ module RequestLogAnalyzer
     def self.build(arguments)
       options = { }
 
-      options[:database] = arguments[:database] if arguments[:database]
+      # Database command line options
+      options[:database]       = arguments[:database] if arguments[:database]
+      options[:reset_database] = arguments[:reset_database]
+      
       options[:debug]    = arguments[:debug]
       options[:dump]     = arguments[:dump]
 
@@ -40,6 +43,9 @@ module RequestLogAnalyzer
       if arguments[:file]
         output_file = File.new(arguments[:file], "w+")
         options[:output] = output_class.new(output_file, :width => 80, :color => false, :characters => :ascii)
+      elsif arguments[:mail]
+        output_mail = RequestLogAnalyzer::Mailer.new(arguments[:mail])
+        options[:output] = output_class.new(output_mail, :width => 80, :color => false, :characters => :ascii)        
       else
         options[:output] = output_class.new(STDOUT, :width => arguments[:report_width].to_i, 
             :color => !arguments[:boring], :characters => (arguments[:boring] ? :ascii : :utf))
@@ -68,7 +74,7 @@ module RequestLogAnalyzer
       end
       
       controller = Controller.new(RequestLogAnalyzer::Source::LogParser.new(file_format, options), options)
-      #controller = Controller.new(RequestLogAnalyzer::Source::Database.new(file_format, options), options)
+      #controller = Controller.new(RequestLogAnalyzer::Source::DatabaseLoader.new(file_format, options), options)
 
       options[:parse_strategy] = arguments[:parse_strategy]
       
@@ -92,9 +98,9 @@ module RequestLogAnalyzer
       arguments[:aggregator].each { |agg| controller.add_aggregator(agg.to_sym) }
 
       # register the database 
-      controller.add_aggregator(:database)   if arguments[:database] && !arguments[:aggregator].include?('database')
-      controller.add_aggregator(:summarizer) if arguments[:aggregator].empty?
-    
+      controller.add_aggregator(:summarizer)          if arguments[:aggregator].empty?
+      controller.add_aggregator(:database_inserter)   if arguments[:database] && !arguments[:aggregator].include?('database')
+
       # register the echo aggregator in debug mode
       controller.add_aggregator(:echo) if arguments[:debug]
       
@@ -128,6 +134,8 @@ module RequestLogAnalyzer
 
       # Handle progress messagess
       @source.progress = lambda { |message, value| handle_progress(message, value) } if @source
+      
+      @source.source_changes = lambda { |change, filename| handle_source_change(change, filename) } if @source
     end
     
     # Progress function.
@@ -151,6 +159,11 @@ module RequestLogAnalyzer
       end
     end
     
+    # Source change handler
+    def handle_source_change(change, filename)
+      @aggregators.each { |agg| agg.source_change(change, File.expand_path(filename, Dir.pwd)) }
+    end
+    
     # Adds an aggregator to the controller. The aggregator will be called for every request 
     # that is parsed from the provided sources (see add_source)
     def add_aggregator(agg)      
@@ -180,8 +193,9 @@ module RequestLogAnalyzer
     # Push a request to all the aggregators (@aggregators).
     # <tt>request</tt> The request to push to the aggregators.    
     def aggregate_request(request)
-      return unless request
+      return false unless request
       @aggregators.each { |agg| agg.aggregate(request) }
+      return true
     end
     
     # Runs RequestLogAnalyzer
@@ -198,8 +212,8 @@ module RequestLogAnalyzer
       install_signal_handlers
       
       @source.each_request do |request|
-        aggregate_request(filter_request(request))
         break if @interrupted
+        aggregate_request(filter_request(request))
       end
 
       @aggregators.each { |agg| agg.finalize }
@@ -217,6 +231,8 @@ module RequestLogAnalyzer
         puts "Mail to contact@railsdoctors.com or visit us at http://railsdoctors.com"
         puts "Thanks for using request-log-analyzer!"
         @output.io.close
+      elsif @output.io.kind_of?(RequestLogAnalyzer::Mailer)
+        @output.io.mail
       end
     end
     

data/lib/request_log_analyzer/database/base.rb

@@ -0,0 +1,114 @@
+class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
+  
+  self.abstract_class = true
+
+  def <=>(other)
+    if (source_id.nil? && other.source_id.nil?) || (source_comparison = source_id <=> other.source_id) == 0
+      lineno <=> other.lineno
+    else
+      source_comparison
+    end
+  end
+
+  def line_type
+    self.class.name.underscore.gsub(/_line$/, '').to_sym
+  end
+
+  cattr_accessor :database, :line_definition
+
+  def self.subclass_from_line_definition(definition)
+    klass = Class.new(RequestLogAnalyzer::Database::Base)
+    klass.set_table_name("#{definition.name}_lines")
+    
+    klass.line_definition = definition
+    
+    # Set relations with requests and sources table
+    klass.belongs_to :request
+    klass.belongs_to :source
+    
+    # Serialize complex fields into the database
+    definition.captures.select { |c| c.has_key?(:provides) }.each do |capture|
+      klass.send(:serialize, capture[:name], Hash)
+    end
+
+    database.request_class.has_many  "#{definition.name}_lines".to_sym
+    database.source_class.has_many   "#{definition.name}_lines".to_sym
+
+    return klass
+  end
+
+  def self.subclass_from_table(table)
+    raise "Table #{table} not found!" unless database.connection.table_exists?(table)
+
+    klass = Class.new(RequestLogAnalyzer::Database::Base)
+    klass.set_table_name(table)
+
+    if klass.column_names.include?('request_id')
+      klass.belongs_to :request
+      database.request_class.has_many table.to_sym
+    end
+    
+    if klass.column_names.include?('source_id')
+      klass.belongs_to :source
+      database.source_class.has_many table.to_sym
+    end
+    
+    return klass
+  end
+  
+  def self.drop_table!
+    database.connection.remove_index(self.table_name, [:source_id])  rescue nil
+    database.connection.remove_index(self.table_name, [:request_id]) rescue nil
+    database.connection.drop_table(self.table_name) if database.connection.table_exists?(self.table_name)
+  end
+  
+  def self.create_table!
+    raise "No line_definition available to base table schema on!" unless self.line_definition
+    
+    unless table_exists?
+      database.connection.create_table(table_name.to_sym) do |t|
+      
+        # Default fields
+        t.column :request_id, :integer
+        t.column :source_id,  :integer
+        t.column :lineno,     :integer
+      
+        line_definition.captures.each do |capture|
+          # Add a field for every capture
+          t.column(capture[:name], column_type(capture[:type]))
+
+          # If the capture provides other field as well, create columns for them, too
+          capture[:provides].each { |field, field_type| t.column(field, column_type(field_type)) } if capture[:provides].kind_of?(Hash)
+        end
+      end
+    end
+    
+    # Add indices to table for more speedy querying
+    database.connection.add_index(self.table_name.to_sym, [:request_id]) # rescue
+    database.connection.add_index(self.table_name.to_sym, [:source_id])  # rescue
+  end
+  
+  
+  # Function to determine the column type for a field
+  # TODO: make more robust / include in file-format definition
+  def self.column_type(type_indicator)
+    case type_indicator
+    when :eval;      :text
+    when :hash;      :text
+    when :text;      :text
+    when :string;    :string
+    when :sec;       :double
+    when :msec;      :double
+    when :duration;  :double
+    when :float;     :double
+    when :double;    :double
+    when :integer;   :integer
+    when :int;       :int
+    when :timestamp; :datetime
+    when :datetime;  :datetime
+    when :date;      :date
+    else             :string
+    end
+  end  
+  
+end

data/lib/request_log_analyzer/database/connection.rb

@@ -0,0 +1,38 @@
+module RequestLogAnalyzer::Database::Connection
+
+  def self.from_string(string)
+    hash = {}
+    if string =~ /^(?:\w+=(?:[^;])*;)*\w+=(?:[^;])*$/
+      string.scan(/(\w+)=([^;]*);?/) { |variable, value| hash[variable.to_sym] = value }
+    elsif string =~ /^(\w+)\:\/\/(?:(?:([^:]+)(?:\:([^:]+))?\@)?([\w\.-]+)\/)?([\w\:\-\.\/]+)$/
+      hash[:adapter], hash[:username], hash[:password], hash[:host], hash[:database] = $1, $2, $3, $4, $5
+      hash.delete_if { |k, v| v.nil? }
+    end
+    return hash.empty? ? nil : hash
+  end
+
+  def connect(connection_identifier)
+    if connection_identifier.kind_of?(Hash)
+      RequestLogAnalyzer::Database::Base.establish_connection(connection_identifier)
+    elsif connection_identifier == ':memory:'
+      RequestLogAnalyzer::Database::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:')
+    elsif connection_hash = RequestLogAnalyzer::Database::Connection.from_string(connection_identifier)
+      RequestLogAnalyzer::Database::Base.establish_connection(connection_hash)
+    elsif connection_identifier.kind_of?(String) # Normal SQLite 3 database file
+      RequestLogAnalyzer::Database::Base.establish_connection(:adapter => 'sqlite3', :database => connection_identifier)
+    elsif connection_identifier.nil?
+      nil
+    else
+      raise "Cannot connect with this connection_identifier: #{connection_identifier.inspect}"
+    end
+  end
+  
+  def disconnect
+    RequestLogAnalyzer::Database::Base.remove_connection
+  end
+  
+  def connection
+    RequestLogAnalyzer::Database::Base.connection
+  end
+  
+end