wvanbergen-request-log-analyzer 0.3.3 → 0.3.4

data/Rakefile

@@ -1,5 +1,5 @@
 Dir[File.dirname(__FILE__) + "/tasks/*.rake"].each { |file| load(file) }
  
-desc 'Default: run RSpec for request-log-analyzer.'
-task :default => :spec
+desc 'Default: run unit tests for request-log-analyzer.'
+task :default => :test
 

data/TODO

@@ -19,7 +19,6 @@ Other:
 
 
 
-
 Datamining should look something like this:
 
 > request-log-analyzer myapp.log --interactive

data/bin/request-log-analyzer

@@ -7,17 +7,14 @@ puts "Request log analyzer, by Willem van Bergen and Bart ten Brinke\n\n"
 # Parse the arguments given via commandline
 begin
   arguments = CommandLine::Arguments.parse do |command_line|
-    
-    #command_line.flag(:install, :alias => :i) # command_line.command(:install)
-    
-    command_line.flag(:format, :default => 'rails')
-    command_line.flag(:aggregator, :alias => :a, :multiple => true)
-    command_line.flag(:database, :alias => :d)
-        
-    command_line.switch(:combined_requests, :c)
-    command_line.switch(:colorize, :z) 
-    #command_line.switch(:estimate_database_time, :e) 
-    #command_line.switch(:fast, :f) #
+    command_line.switch(:guess_database_time, :g)
+    command_line.switch(:fast, :f)
+    command_line.switch(:colorize, :z)
+    command_line.switch(:merb, :m)
+    command_line.switch(:install, :i)
+    command_line.flag(:output, :alias => :o)
+    command_line.flag(:amount, :alias => :c)
+    command_line.required_files = 1
   end
   
 rescue CommandLine::Error => e
@@ -27,22 +24,25 @@ rescue CommandLine::Error => e
   exit(0) 
 end
 
-# if arguments[:install]
-#   if arguments[:install] == 'rails'
-#     require 'ftools'
-#     if File.directory?('./lib/tasks/')
-#       File.copy(File.dirname(__FILE__) + '/../tasks/request_log_analyzer.rake', './lib/tasks/request_log_analyze.rake')
-#       puts "Installed rake tasks."
-#       puts "To use, run: rake log:analyze"
-#     else
-#       puts "Cannot find /lib/tasks folder. Are you in your Rails directory?"
-#       puts "Installation aborted."
-#     end
-#   else
-#     raise "Cannot perform this install type!"
-#   end
-#   exit(0)
-# end
+if arguments[:install]
+  if arguments.files.first == 'rails'
+    require 'ftools'
+    if File.directory?('./lib/tasks/')
+      File.copy(File.dirname(__FILE__) + '/../tasks/request_log_analyzer.rake', './lib/tasks/request_log_analyze.rake')
+      puts "Installed rake tasks."
+      puts "To use, run: rake log:analyze"
+    else
+      puts "Cannot find /lib/tasks folder. Are you in your Rails directory?"
+      puts "Installation aborted."
+    end
+  end
+  exit(0)
+end
+
+$colorize = true if arguments[:colorize]
 
 # Run the request_log_analyzer!
-request_log_analyzer = RequestLogAnalyzer::Controller.build(arguments).run!
+request_log_analyzer = RequestLogAnalyzer.new(arguments)
+request_log_analyzer.analyze_this(arguments.files)
+
+

data/bin/request-log-database

@@ -0,0 +1,81 @@
+#!/usr/bin/ruby
+
+require File.dirname(__FILE__) + '/../lib/command_line/arguments'
+require File.dirname(__FILE__) + '/../lib/base/log_parser'
+require File.dirname(__FILE__) + '/../lib/base/record_inserter'
+require File.dirname(__FILE__) + '/../lib/rails_analyzer/log_parser'
+require File.dirname(__FILE__) + '/../lib/rails_analyzer/record_inserter'
+require File.dirname(__FILE__) + '/../lib/bashcolorizer'
+require File.dirname(__FILE__) + '/../lib/ruby-progressbar/progressbar.rb'
+
+
+puts "Rails log parser, by Willem van Bergen and Bart ten Brinke\n\n"
+
+begin
+  
+  $arguments = CommandLine::Arguments.parse do |command_line|
+    command_line.switch(:guess_database_time, :g)
+    command_line.switch(:reset_database, :r)
+    command_line.flag(:database, :alias => :d, :required => false)
+    command_line.required_files = 1
+  end
+  
+rescue CommandLine::Error => e
+  puts "ARGUMENT ERROR: " + e.message
+  puts
+  puts "Usage: ruby parsetodb.rb [LOGFILES*] <OPTIONS>"
+  puts
+  puts "Options:"
+  puts "  --database, -t:             The database file to use"
+  puts "  --reset-database, -r:       Resets the database before inserting new records"
+  puts "  --guess-database-time, -g:  Guesses the database duration of requests"      
+  puts
+  puts "Examples:"
+  puts "  ./parsetodb.rb development.log"
+  puts "  ./parsetodb.rb mongrel.0.log mongrel.1.log mongrel.2.log -g -d mongrel.db"  
+  puts 
+   
+  exit(0) 
+end
+
+log_files  = $arguments.files
+db_file    = $arguments[:database] || log_files.first + '.db'
+
+if $arguments[:reset_database] && File.exist?(db_file) 
+  File.delete(db_file) 
+  puts "Database file cleared."
+end
+
+records_inserted = 0 
+inserter = RailsAnalyzer::RecordInserter.insert_batch_into(db_file) do |db|
+  log_files.each do |log_file|
+
+    puts "Processing all log lines from #{log_file}..."
+    parser = RailsAnalyzer::LogParser.new(log_file)
+    
+    pbar = ProgressBar.new(green(log_file), File.size(log_file))
+    parser.progress { |pos, total| (pos == :finished) ? pbar.finish : pbar.set(pos) }
+    
+    parser.each do |request| 
+      db.insert(request) 
+      records_inserted += 1
+    end
+  end
+  
+  if $arguments[:guess_database_time]
+    puts "Calculating database times..."
+    db.calculate_db_durations! 
+  end
+end
+
+started   = inserter.count(:started)
+completed = inserter.count(:completed)
+failed    = inserter.count(:failed)
+
+puts 
+puts "Inserted #{records_inserted} records from #{log_files.length} files."
+puts "Parse warnings: #{inserter.warning_count}. Check the parse_warnings table in the database for details."
+puts
+puts "Requests started: #{started}"
+puts "Requests completed: #{completed}"
+puts "Requests failed: #{failed}"

data/lib/base/log_parser.rb

@@ -0,0 +1,78 @@
+module Base
+  # Parse a log file
+  class LogParser 
+
+    LOG_LINES = {}
+  
+    # LogParser initializer
+    # <tt>file</tt> The fileobject this LogParser wil operate on.
+    def initialize(file, options = {})
+      @file_name = file
+      @options = options
+      @file_size = File.size(@file_name)
+      
+      self.initialize_hook(options) if self.respond_to?(:initialize_hook)
+    end
+  
+    def progress(&block)
+      @progress_handler = block
+    end
+
+    # Output a warning
+    # <tt>message</tt> The warning message (object)
+    def warn(message)
+      puts " -> " + message.to_s
+    end  
+
+    def convert_value(value, type) 
+      return case type
+      when :string;    value.to_s
+      when :int;       value.to_i
+      when :sec;       value.to_f
+      when :msec;      value.to_f / 1000
+      when :timestamp; value.to_s # TODO: fix me?
+      else
+        warn("Unkwn type encountered: #{type}")
+        value
+      end
+    end
+
+    # Finds a log line and then parses the information in the line.
+    # Yields a hash containing the information found. 
+    # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
+    # Yeilds a Hash when it encounters a chunk of information.
+    def each(*line_types, &block)
+      log_lines_hash = self.class::LOG_LINES
+
+      
+      # parse everything by default 
+      line_types = log_lines_hash.keys if line_types.empty?
+
+      File.open(@file_name) do |file|
+      
+        file.each_line do |line|
+        
+          @progress_handler.call(file.pos, @file_size) if @progress_handler
+          
+          line_types.each do |line_type|
+            if log_lines_hash[line_type][:teaser] =~ line
+              if log_lines_hash[line_type][:regexp] =~ line
+                
+                captures = $~.captures
+                request = { :type => line_type, :line => file.lineno }
+                log_lines_hash[line_type][:params].each_with_index do |paramhash, index|              
+                  paramhash.each { |key, type| request[key] = convert_value(captures[index], type) } unless captures[index].nil?
+                end
+                
+                yield(request) if block_given?
+              else
+                warn("Unparsable #{line_type} line: " + line[0..79]) unless line_type == :failed
+              end
+            end            
+          end
+        end
+        @progress_handler.call(:finished, @file_size) if @progress_handler
+      end      
+    end
+  end
+end

data/lib/base/record_inserter.rb

@@ -0,0 +1,139 @@
+require 'rubygems'
+require 'sqlite3'
+
+module Base
+  
+  # Set of functions that can be used to easily log requests into a SQLite3 Database.
+  class RecordInserter
+    
+    attr_reader :database
+    attr_reader :current_request
+    attr_reader :warning_count
+    
+    # Initializer
+    # <tt>db_file</tt> The file which will be used for the SQLite3 Database storage.
+    def initialize(db_file, options = {})
+      @database = SQLite3::Database.new(db_file)
+      @insert_statements = nil
+      @warning_count = 0
+      create_tables_if_needed!
+
+      self.initialize_hook(options) if self.respond_to?(:initialize_hook)
+    end
+        
+    # Calculate the database durations of the requests currenty in the database.
+    # Used if a logfile does contain any database durations.
+    def calculate_db_durations!
+      @database.execute('UPDATE "completed_queries" SET "database" = "duration" - "rendering" WHERE "database" IS NULL OR "database" = 0.0')
+    end
+    
+    # Insert a batch of loglines into the database.
+    # Function prepares insert statements, yeilds and then closes and commits.
+    def insert_batch(&block)
+      @database.transaction
+      prepare_statements!
+      block.call(self)
+      close_prepared_statements!
+      @database.commit
+    rescue Exception => e
+      puts e.message
+      @database.rollback
+    end
+    
+    def insert_warning(line, warning)
+      @database.execute("INSERT INTO parse_warnings (line, warning) VALUES (:line, :warning)", :line => line, :warning => warning)
+      @warning_count += 1
+    end
+        
+    # Insert a request into the database.
+    # def insert(request, close_statements = false)
+    #   raise 'No insert defined for this log file type'
+    # end
+    
+    # Insert a batch of files into the database.
+    # <tt>db_file</tt> The filename of the database file to use.
+    # Returns the created database.
+    def self.insert_batch_into(db_file, options = {}, &block)
+      db = self.new(db_file)
+      db.insert_batch(&block)
+      return db
+    end    
+    
+    def count(type)
+      @database.get_first_value("SELECT COUNT(*) FROM \"#{type}_requests\"").to_i
+    end
+        
+    protected
+    
+    # Prepare insert statements.
+    def prepare_statements!
+      @insert_statements = {
+        :started => @database.prepare("
+            INSERT INTO started_requests ( line,  timestamp,  ip,  method,  controller,  action) 
+                                  VALUES (:line, :timestamp, :ip, :method, :controller, :action)"),
+                                  
+        :failed => @database.prepare("
+            INSERT INTO failed_requests ( line,  exception_string,  stack_trace,  error)
+                                 VALUES (:line, :exception_string, :stack_trace, :error)"),
+                                 
+        :completed => @database.prepare("
+            INSERT INTO completed_requests ( line,  url,  status,  duration,  rendering_time,  database_time)
+                                    VALUES (:line, :url, :status, :duration, :rendering, :db)")
+      }
+    end
+    
+    # Close all prepared statments
+    def close_prepared_statements!
+      @insert_statements.each { |key, stmt| stmt.close }
+    end
+
+    # Create the needed database tables if they don't exist.
+    def create_tables_if_needed!
+      
+      @database.execute("
+        CREATE TABLE IF NOT EXISTS started_requests (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          line INTEGER NOT NULL,
+          timestamp DATETIME NOT NULL, 
+          controller VARCHAR(255) NOT NULL, 
+          action VARCHAR(255) NOT NULL,
+          method VARCHAR(6) NOT NULL,          
+          ip VARCHAR(6) NOT NULL
+        )
+      ");
+
+      @database.execute("
+          CREATE TABLE IF NOT EXISTS failed_requests (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            line INTEGER NOT NULL,            
+            started_request_id INTEGER,        
+            error VARCHAR(255),    
+            exception_string VARCHAR(255),
+            stack_trace TEXT
+          )      
+      ");
+
+      @database.execute("
+        CREATE TABLE IF NOT EXISTS completed_requests (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          line INTEGER NOT NULL,          
+          started_request_id INTEGER,
+          url VARCHAR(255) NOT NULL,
+          hashed_url VARCHAR(255),
+          status INTEGER NOT NULL,
+          duration FLOAT,
+          rendering_time FLOAT,
+          database_time FLOAT
+        )
+      ");    
+      
+      @database.execute("CREATE TABLE IF NOT EXISTS parse_warnings (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          line INTEGER NOT NULL,       
+          warning VARCHAR(255) NOT NULL
+        )
+      ");
+    end
+
+  end
+end

data/lib/command_line/arguments.rb

@@ -46,7 +46,7 @@ module CommandLine
     # <tt>flag</tt> A flag symbol like :fast
     # Options
     # * <tt>:expects</tt> Expects a value after the flag
-    def flag(flag, options = {})
+    def flag(flag, options)
       options[:expects] = String unless options.has_key?(:expects)
       argument = Flag.new(flag, options)
       @flag_definitions[argument.to_argument]  = argument
@@ -85,14 +85,7 @@ module CommandLine
             if flag.expects_argument?
               
               if @arguments.length > (i + 1) && @arguments[i + 1]
-                
-                if flag.multiple?
-                  @flags[flag.name] ||= []
-                  @flags[flag.name] << @arguments[i + 1]
-                else
-                  @flags[flag.name] = @arguments[i + 1]
-                end
-                                
+                @flags[flag.name] = @arguments[i + 1]
                 i += 1
               else
                 raise CommandLine::FlagExpectsArgument.new(arg)
@@ -122,12 +115,6 @@ module CommandLine
     # Check if the parsed arguments meet their requirements.
     # Raises CommandLineexception on error.
     def check_parsed_arguments!
-      
-      @flag_definitions.each do |flag, definition| 
-        @flags[definition.name] ||= [] if definition.multiple? && !definition.default?
-        @flags[definition.name] ||= definition.default if definition.default?
-      end
-
       if @begins_with_command && @command.nil? 
         raise CommandLine::CommandMissing.new
       end

data/lib/command_line/flag.rb

@@ -6,8 +6,6 @@ module CommandLine
     attr_reader :name
     attr_reader :alias
     attr_reader :argument
-    attr_reader :default
-    attr_reader :multiple
     
     # Initialize new Flag
     # <tt>name</tt> The name of the flag
@@ -17,8 +15,6 @@ module CommandLine
       @alias    = definition[:alias].to_sym if definition[:alias]
       @required = definition.has_key?(:required) && definition[:required] == true
       @argument = definition[:expects] if definition[:expects]
-      @multiple = definition[:multiple] || false
-      @default  = definition[:default] if definition[:default]
     end
     
     # Argument representation of the flag (--fast)
@@ -41,14 +37,6 @@ module CommandLine
       !@required
     end
     
-    def multiple?
-      @multiple
-    end
-    
-    def default?
-      !@default.nil?
-    end
-    
     # Check if flag is required
     def required?
       @required