wvanbergen-request-log-analyzer 0.2.2 → 0.3.0

data/bin/request-log-database

@@ -1,81 +0,0 @@
-#!/usr/bin/ruby
-
-require File.dirname(__FILE__) + '/../lib/command_line/arguments'
-require File.dirname(__FILE__) + '/../lib/base/log_parser'
-require File.dirname(__FILE__) + '/../lib/base/record_inserter'
-require File.dirname(__FILE__) + '/../lib/rails_analyzer/log_parser'
-require File.dirname(__FILE__) + '/../lib/rails_analyzer/record_inserter'
-require File.dirname(__FILE__) + '/../lib/bashcolorizer'
-require File.dirname(__FILE__) + '/../lib/ruby-progressbar/progressbar.rb'
-
-
-puts "Rails log parser, by Willem van Bergen and Bart ten Brinke\n\n"
-
-begin
-  
-  $arguments = CommandLine::Arguments.parse do |command_line|
-    command_line.switch(:guess_database_time, :g)
-    command_line.switch(:reset_database, :r)
-    command_line.flag(:database, :alias => :d, :required => false)
-    command_line.required_files = 1
-  end
-  
-rescue CommandLine::Error => e
-  puts "ARGUMENT ERROR: " + e.message
-  puts
-  puts "Usage: ruby parsetodb.rb [LOGFILES*] <OPTIONS>"
-  puts
-  puts "Options:"
-  puts "  --database, -t:             The database file to use"
-  puts "  --reset-database, -r:       Resets the database before inserting new records"
-  puts "  --guess-database-time, -g:  Guesses the database duration of requests"      
-  puts
-  puts "Examples:"
-  puts "  ./parsetodb.rb development.log"
-  puts "  ./parsetodb.rb mongrel.0.log mongrel.1.log mongrel.2.log -g -d mongrel.db"  
-  puts 
-   
-  exit(0) 
-end
-
-log_files  = $arguments.files
-db_file    = $arguments[:database] || log_files.first + '.db'
-
-if $arguments[:reset_database] && File.exist?(db_file) 
-  File.delete(db_file) 
-  puts "Database file cleared."
-end
-
-records_inserted = 0 
-inserter = RailsAnalyzer::RecordInserter.insert_batch_into(db_file) do |db|
-  log_files.each do |log_file|
-
-    puts "Processing all log lines from #{log_file}..."
-    parser = RailsAnalyzer::LogParser.new(log_file)
-    
-    pbar = ProgressBar.new(green(log_file), File.size(log_file))
-    parser.progress { |pos, total| (pos == :finished) ? pbar.finish : pbar.set(pos) }
-    
-    parser.each do |request| 
-      db.insert(request) 
-      records_inserted += 1
-    end
-  end
-  
-  if $arguments[:guess_database_time]
-    puts "Calculating database times..."
-    db.calculate_db_durations! 
-  end
-end
-
-started   = inserter.count(:started)
-completed = inserter.count(:completed)
-failed    = inserter.count(:failed)
-
-puts 
-puts "Inserted #{records_inserted} records from #{log_files.length} files."
-puts "Parse warnings: #{inserter.warning_count}. Check the parse_warnings table in the database for details."
-puts
-puts "Requests started: #{started}"
-puts "Requests completed: #{completed}"
-puts "Requests failed: #{failed}"

data/lib/base/log_parser.rb

@@ -1,78 +0,0 @@
-module Base
-  # Parse a log file
-  class LogParser 
-
-    LOG_LINES = {}
-  
-    # LogParser initializer
-    # <tt>file</tt> The fileobject this LogParser wil operate on.
-    def initialize(file, options = {})
-      @file_name = file
-      @options = options
-      @file_size = File.size(@file_name)
-      
-      self.initialize_hook(options) if self.respond_to?(:initialize_hook)
-    end
-  
-    def progress(&block)
-      @progress_handler = block
-    end
-
-    # Output a warning
-    # <tt>message</tt> The warning message (object)
-    def warn(message)
-      puts " -> " + message.to_s
-    end  
-
-    def convert_value(value, type) 
-      return case type
-      when :string;    value.to_s
-      when :int;       value.to_i
-      when :sec;       value.to_f
-      when :msec;      value.to_f / 1000
-      when :timestamp; value.to_s # TODO: fix me?
-      else
-        warn("Unkwn type encountered: #{type}")
-        value
-      end
-    end
-
-    # Finds a log line and then parses the information in the line.
-    # Yields a hash containing the information found. 
-    # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
-    # Yeilds a Hash when it encounters a chunk of information.
-    def each(*line_types, &block)
-      log_lines_hash = self.class::LOG_LINES
-
-      
-      # parse everything by default 
-      line_types = log_lines_hash.keys if line_types.empty?
-
-      File.open(@file_name) do |file|
-      
-        file.each_line do |line|
-        
-          @progress_handler.call(file.pos, @file_size) if @progress_handler
-          
-          line_types.each do |line_type|
-            if log_lines_hash[line_type][:teaser] =~ line
-              if log_lines_hash[line_type][:regexp] =~ line
-                
-                captures = $~.captures
-                request = { :type => line_type, :line => file.lineno }
-                log_lines_hash[line_type][:params].each_with_index do |paramhash, index|              
-                  paramhash.each { |key, type| request[key] = convert_value(captures[index], type) } unless captures[index].nil?
-                end
-                
-                yield(request) if block_given?
-              else
-                warn("Unparsable #{line_type} line: " + line[0..79]) unless line_type == :failed
-              end
-            end            
-          end
-        end
-        @progress_handler.call(:finished, @file_size) if @progress_handler
-      end      
-    end
-  end
-end

data/lib/base/record_inserter.rb

@@ -1,139 +0,0 @@
-require 'rubygems'
-require 'sqlite3'
-
-module Base
-  
-  # Set of functions that can be used to easily log requests into a SQLite3 Database.
-  class RecordInserter
-    
-    attr_reader :database
-    attr_reader :current_request
-    attr_reader :warning_count
-    
-    # Initializer
-    # <tt>db_file</tt> The file which will be used for the SQLite3 Database storage.
-    def initialize(db_file, options = {})
-      @database = SQLite3::Database.new(db_file)
-      @insert_statements = nil
-      @warning_count = 0
-      create_tables_if_needed!
-
-      self.initialize_hook(options) if self.respond_to?(:initialize_hook)
-    end
-        
-    # Calculate the database durations of the requests currenty in the database.
-    # Used if a logfile does contain any database durations.
-    def calculate_db_durations!
-      @database.execute('UPDATE "completed_queries" SET "database" = "duration" - "rendering" WHERE "database" IS NULL OR "database" = 0.0')
-    end
-    
-    # Insert a batch of loglines into the database.
-    # Function prepares insert statements, yeilds and then closes and commits.
-    def insert_batch(&block)
-      @database.transaction
-      prepare_statements!
-      block.call(self)
-      close_prepared_statements!
-      @database.commit
-    rescue Exception => e
-      puts e.message
-      @database.rollback
-    end
-    
-    def insert_warning(line, warning)
-      @database.execute("INSERT INTO parse_warnings (line, warning) VALUES (:line, :warning)", :line => line, :warning => warning)
-      @warning_count += 1
-    end
-        
-    # Insert a request into the database.
-    # def insert(request, close_statements = false)
-    #   raise 'No insert defined for this log file type'
-    # end
-    
-    # Insert a batch of files into the database.
-    # <tt>db_file</tt> The filename of the database file to use.
-    # Returns the created database.
-    def self.insert_batch_into(db_file, options = {}, &block)
-      db = self.new(db_file)
-      db.insert_batch(&block)
-      return db
-    end    
-    
-    def count(type)
-      @database.get_first_value("SELECT COUNT(*) FROM \"#{type}_requests\"").to_i
-    end
-        
-    protected
-    
-    # Prepare insert statements.
-    def prepare_statements!
-      @insert_statements = {
-        :started => @database.prepare("
-            INSERT INTO started_requests ( line,  timestamp,  ip,  method,  controller,  action) 
-                                  VALUES (:line, :timestamp, :ip, :method, :controller, :action)"),
-                                  
-        :failed => @database.prepare("
-            INSERT INTO failed_requests ( line,  exception_string,  stack_trace,  error)
-                                 VALUES (:line, :exception_string, :stack_trace, :error)"),
-                                 
-        :completed => @database.prepare("
-            INSERT INTO completed_requests ( line,  url,  status,  duration,  rendering_time,  database_time)
-                                    VALUES (:line, :url, :status, :duration, :rendering, :db)")
-      }
-    end
-    
-    # Close all prepared statments
-    def close_prepared_statements!
-      @insert_statements.each { |key, stmt| stmt.close }
-    end
-
-    # Create the needed database tables if they don't exist.
-    def create_tables_if_needed!
-      
-      @database.execute("
-        CREATE TABLE IF NOT EXISTS started_requests (
-          id INTEGER PRIMARY KEY AUTOINCREMENT,
-          line INTEGER NOT NULL,
-          timestamp DATETIME NOT NULL, 
-          controller VARCHAR(255) NOT NULL, 
-          action VARCHAR(255) NOT NULL,
-          method VARCHAR(6) NOT NULL,          
-          ip VARCHAR(6) NOT NULL
-        )
-      ");
-
-      @database.execute("
-          CREATE TABLE IF NOT EXISTS failed_requests (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            line INTEGER NOT NULL,            
-            started_request_id INTEGER,        
-            error VARCHAR(255),    
-            exception_string VARCHAR(255),
-            stack_trace TEXT
-          )      
-      ");
-
-      @database.execute("
-        CREATE TABLE IF NOT EXISTS completed_requests (
-          id INTEGER PRIMARY KEY AUTOINCREMENT,
-          line INTEGER NOT NULL,          
-          started_request_id INTEGER,
-          url VARCHAR(255) NOT NULL,
-          hashed_url VARCHAR(255),
-          status INTEGER NOT NULL,
-          duration FLOAT,
-          rendering_time FLOAT,
-          database_time FLOAT
-        )
-      ");    
-      
-      @database.execute("CREATE TABLE IF NOT EXISTS parse_warnings (
-          id INTEGER PRIMARY KEY AUTOINCREMENT,
-          line INTEGER NOT NULL,       
-          warning VARCHAR(255) NOT NULL
-        )
-      ");
-    end
-
-  end
-end

data/lib/merb_analyzer/log_parser.rb

@@ -1,26 +0,0 @@
-module MerbAnalyzer
-
-  class LogParser < Base::LogParser
-    LOG_LINES = {
-      # ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
-      :started => {
-        :teaser => /Started/,
-        :regexp => /Started request handling\:\ (.+)/,
-        :params => [{:timestamp => :timestamp}]
-      },
-      # ~ Params: {"action"=>"create", "controller"=>"session"}
-      # ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
-      :params => {
-        :teaser => /Params/,
-        :regexp => /Params\:\ \{(.+)\}/,
-        :params => [{:raw_hash => :string}]
-      },
-      # ~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
-      :completed => {
-        :teaser => /\{:dispatch_time/,
-        :regexp => /\{\:dispatch_time=>(.+), (?:\:after_filters_time=>(.+), )?(?:\:before_filters_time=>(.+), )?\:action_time=>(.+)\}/,
-        :params => [ {:dispatch_time => :sec}, {:after_filters_time => :sec}, {:before_filters_time => :sec}, {:action_time => :sec} ]
-      }
-    }
-  end
-end

data/lib/rails_analyzer/log_parser.rb

@@ -1,35 +0,0 @@
-module RailsAnalyzer
-
-  class LogParser < Base::LogParser
-    
-    # Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
-    RAILS_21_COMPLETED = /Completed in (\d+\.\d{5}) \(\d+ reqs\/sec\) (?:\| Rendering: (\d+\.\d{5}) \(\d+\%\) )?(?:\| DB: (\d+\.\d{5}) \(\d+\%\) )?\| (\d\d\d).+\[(http.+)\]/
-
-    # Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]  
-    RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+), )?DB: (\d+)\) \| (\d\d\d).+\[(http.+)\]/
-    
-    
-    LOG_LINES = {
-      # Processing EmployeeController#index (for 123.123.123.123 at 2008-07-13 06:00:00) [GET]
-      :started => {
-        :teaser => /Processing/,
-        :regexp => /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/,
-        :params => [{:controller => :string}, {:action => :string}, {:format => :string}, {:ip => :string}, {:timestamp => :timestamp}, {:method => :string}]
-      },
-      # RuntimeError (Cannot destroy employee):  /app/models/employee.rb:198:in `before_destroy' 
-      :failed => {
-        :teaser => /Error/,
-        :regexp => /(\w+)(?:Error|Invalid) \((.*)\)\:(.*)/,
-        :params => [{:error => :string}, {:exception_string => :string}, {:stack_trace => :string}]
-      },
-    
-      :completed => {
-        :teaser   => /Completed in /,
-        :regexp   => Regexp.new("(?:#{RAILS_21_COMPLETED}|#{RAILS_22_COMPLETED})"),
-        :params   => [{:duration => :sec},  {:rendering => :sec},  {:db => :sec},  {:status => :int}, {:url => :string},  # 2.1 variant 
-                      {:duration => :msec}, {:rendering => :msec}, {:db => :msec}, {:status => :int}, {:url => :string}]  # 2.2 variant 
-
-      }
-    }
-  end
-end

data/lib/rails_analyzer/record_inserter.rb

@@ -1,39 +0,0 @@
-require 'rubygems'
-require 'sqlite3'
-
-module RailsAnalyzer
-  
-  # Set of functions that can be used to easily log requests into a SQLite3 Database.
-  class RecordInserter < Base::RecordInserter
-    
-    # Insert a request into the database.
-    # <tt>request</tt> The request to insert.
-    # <tt>close_statements</tt> Close prepared statements (default false)
-    def insert(request, close_statements = false)
-      unless @insert_statements
-        prepare_statements! 
-        close_statements = true
-      end
-        
-      if request[:type] && @insert_statements.has_key?(request[:type]) 
-        if request[:type] == :started
-          insert_warning(request[:line], "Unclosed request encountered on line #{request[:line]} (request started on line #{@current_request})") unless @current_request.nil?
-          @current_request = request[:line]
-        elsif [:failed, :completed].include?(request[:type])
-          @current_request = nil
-        end
-        
-        begin
-          @insert_statements[request.delete(:type)].execute(request) 
-        rescue SQLite3::Exception => e
-          insert_warning(request[:line], "Could not save log line to database: " + e.message.to_s)
-        end        
-      else
-        insert_warning(request[:line], "Ignored unknown statement type")
-      end
-      
-      close_prepared_statements! if close_statements
-    end
-    
-  end
-end

data/test/merb_log_parser_test.rb

@@ -1,39 +0,0 @@
-require 'test/unit'
-
-require "#{File.dirname(__FILE__)}/../lib/base/log_parser"
-require "#{File.dirname(__FILE__)}/../lib/merb_analyzer/log_parser"
-
-class MerbLogParserTest < Test::Unit::TestCase
-  
-  def fragment_file(number)
-    "#{File.dirname(__FILE__)}/log_fragments/merb_#{number}.log"
-  end
-  
-  def test_parse_started_merb_fragment
-    requests = []
-    parser = MerbAnalyzer::LogParser.new(fragment_file(1)).each(:started) do |request|
-      requests << request
-    end
-    assert_equal requests[0][:timestamp], "Fri Aug 29 11:10:23 +0200 2008"
-  end
-
-  def test_parse_completed_merb_fragment
-    requests = []
-    parser = MerbAnalyzer::LogParser.new(fragment_file(1)).each(:completed) do |request|
-      requests << request
-    end
-    
-    assert_equal requests[0][:action_time], 0.241652
-  end
-
-  def test_parse_params_merb_fragment
-    requests = []
-    parser = MerbAnalyzer::LogParser.new(fragment_file(1)).each(:params) do |request|
-      requests << request
-    end
-    
-    assert_match '"controller"=>"session"', requests[0][:raw_hash]
-    assert_match '"action"=>"destroy"', requests[0][:raw_hash]
-  end
-  
-end

data/test/rails_log_parser_test.rb

@@ -1,95 +0,0 @@
-require 'test/unit'
-
-require "#{File.dirname(__FILE__)}/../lib/base/log_parser"
-require "#{File.dirname(__FILE__)}/../lib/rails_analyzer/log_parser"
-
-class RailsLogParserTest < Test::Unit::TestCase
-  
-  def fragment_file(number)
-    "#{File.dirname(__FILE__)}/log_fragments/fragment_#{number}.log"
-  end
-  
-  
-  def test_rails_22_log_format
-    count = 0
-    parser = RailsAnalyzer::LogParser.new(fragment_file(3)).each(:completed) do |request|
-      count += 1
-      assert_equal 0.614, request[:duration]  # should be 0.614
-    end
-    assert_equal 1, count  
-  end
-  
-  def test_progress_messages
-    log_file = fragment_file(1)
-  
-    finished_encountered = false
-    file_size = File.size(log_file)
-    
-    previous_pos = -1
-    parser = RailsAnalyzer::LogParser.new(log_file)
-    parser.progress do |pos, total|
-      assert_equal file_size, total
-      if pos == :finished
-        finished_encountered = true 
-      else
-        assert pos <= total
-        assert pos > previous_pos
-        previous_pos = pos
-      end
-    end
-    
-    # now parse the file
-    parser.each(:started) { }
-    
-    assert finished_encountered, "A finished event should have been fired"
-  end
-  
-  def test_parse_mongrel_log_fragment
-    count = 0
-    parser = RailsAnalyzer::LogParser.new(fragment_file(1)).each(:started) { count += 1 }
-    assert_equal 4, count
-    
-    count = 0
-    parser = RailsAnalyzer::LogParser.new(fragment_file(1)).each(:completed) { count += 1 }
-    assert_equal 4, count    
-    
-    count = 0
-    parser = RailsAnalyzer::LogParser.new(fragment_file(1)).each(:started, :completed) { count += 1 }
-    assert_equal 8, count    
-  end
-  
-  def test_parse_syslog_fragment
-    count = 0
-    parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:started) { count += 1 }
-    assert_equal 1, count
-    
-    count = 0
-    parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:completed) { count += 1 }
-    assert_equal 1, count    
-    
-    count = 0
-    parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:started, :completed) { count += 1 }
-    assert_equal 2, count    
-  end
-  
-  def test_parse_syslog_fragment_content
-    # this test only works because there is only one requests in the fragment
-    parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:started) do |request|
-      assert_equal "EmployeeController", request[:controller]
-      assert_equal "index", request[:action]      
-      assert_equal "GET", request[:method]
-      assert_equal '10.1.1.33', request[:ip]
-      assert_equal '2008-07-13 06:25:58', request[:timestamp]            
-    end
-  
-    parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:completed) do |request|
-      assert_equal "http://example.com/employee.xml", request[:url]
-      assert_equal 200, request[:status]
-      assert_equal 0.21665, request[:duration]
-      assert_equal 0.00926, request[:rendering]      
-      assert_equal 0.0, request[:db]
-    end
-  
-  end
-  
-end

data/test/record_inserter_test.rb

@@ -1,45 +0,0 @@
-require 'test/unit'
-
-require "#{File.dirname(__FILE__)}/../lib/base/log_parser"
-require "#{File.dirname(__FILE__)}/../lib/base/record_inserter"
-
-require "#{File.dirname(__FILE__)}/../lib/rails_analyzer/log_parser"
-require "#{File.dirname(__FILE__)}/../lib/rails_analyzer/record_inserter"
-
-class RecordInserterTest < Test::Unit::TestCase
-  
-  def fragment_file(number)
-    "#{File.dirname(__FILE__)}/log_fragments/fragment_#{number}.log"
-  end
-  
-  def setup
-    File.delete('_tmp.db') if File.exist?('_tmp.db')
-  end
-  
-  def teardown
-    File.delete('_tmp.db') if File.exist?('_tmp.db')
-  end
-  
-  def test_insert_log_fragment
-    
-    db = RailsAnalyzer::RecordInserter.insert_batch_into('_tmp.db') do |batch|
-      RailsAnalyzer::LogParser.new(fragment_file(1)).each { |request| batch.insert(request) }
-    end
-
-    assert_equal 4, db.database.get_first_value("SELECT COUNT(*) FROM started_requests").to_i
-    assert_equal 4, db.database.get_first_value("SELECT COUNT(*) FROM completed_requests").to_i 
-  end
-  
-  def test_insert_multiple_fragments
-    RailsAnalyzer::RecordInserter.insert_batch_into('_tmp.db') do |batch|
-      RailsAnalyzer::LogParser.new(fragment_file(1)).each { |request| batch.insert(request) }
-    end
-
-    db = RailsAnalyzer::RecordInserter.insert_batch_into('_tmp.db') do |batch|
-      RailsAnalyzer::LogParser.new(fragment_file(2)).each { |request| batch.insert(request) }
-    end
-    assert_equal 5, db.database.get_first_value("SELECT COUNT(*) FROM started_requests").to_i
-    assert_equal 5, db.database.get_first_value("SELECT COUNT(*) FROM completed_requests").to_i    
-  end
-  
-end

data/test/tasks.rake

@@ -1,8 +0,0 @@
-require 'rake/testtask'
- 
-desc 'Unit test request-log-analyzer.'
-Rake::TestTask.new(:test) do |t|
-  t.pattern = 'test/**/*_test.rb'
-  t.verbose = true
-  t.libs << 'test'
-end