lackac-request-log-analyzer 0.1.3

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Willem van Bergen / Bart ten Brinke
+ 
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+ 
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+ 
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,132 @@
+Request log analyzer
+--------------------------------
+
+This is a simple command line tool to analyze request log files of both Rails and
+Merb. Its purpose is to find what actions are best candidates for optimization.
+
+This tool will parse all requests in the logfile and aggregate the
+information. Once it is finished parsing the log file, it will show the
+requests that take op most server time. Different metrics are used (cumulative
+time, average time, blockers, DB time, etc)
+
+
+Installation
+--------------------------------
+gem sources -a http://gems.github.com
+sudo gem install wvanbergen-request-log-analyzer
+
+Usage
+--------------------------------
+
+Usage: request-log-analyzer [FILE] [OPTION]
+Analyze the given log FILE with the given OPTION
+Example: request-log-analyzer mongrel.log
+
+  --fast, -t:                 Only use completed requests
+  --guess-database-time, -g:  Guesses the database duration of requests if they are not in the log
+  --output, -o:               Comma-separated list of reports to show    
+  --amount, -c:               Displays the top <amount> elements in the reports
+  --colorize, -z:             Fancy bash coloring
+
+
+
+Example
+--------------------------------
+
+Note that this example was shortened for your viewing pleasure.
+$ request-log-analyzer /var/log/my_app.log
+
+Request log analyzer, by Willem van Bergen and  Bart ten Brinke
+
+Processing all log lines...
+========================================================================
+Successfully analyzed 58908 requests from log file
+
+Timestamp first request: 2008-07-13T06:25:58+00:00
+Timestamp last request:  2008-07-20T06:18:53+00:00
+Total time analyzed: 7 days
+Methods: DELETE (1%), GET (50%), POST (22%), PUT (25%).
+
+Top 10 most requested actions
+========================================================================
+/overview/:date/                                  : 19359 requests
+/overview/day/:date/                              : 6365 requests
+/overview/:date/set/                              : 5589 requests
+/overview/                                        : 3985 requests
+/clients/:id/                                     : 1976 requests
+........
+
+Top 10 actions by time - cumulative
+========================================================================
+/overview/:date/                                  :   9044.582s [19359 requests]
+/overview/                                        :   8478.767s [3985 requests]
+/overview/:date/set/                              :   3309.041s [5589 requests]
+/clients/:id/products/:id/                        :   1479.911s [924 requests]
+/clients/:id/                                     :    750.080s [1976 requests]
+........
+
+Top 10 actions by time - per request mean
+========================================================================
+/overview/                                        :      2.128s [3985 requests]
+/clients/:id/products/:id/                        :      1.602s [924 requests]
+/overview/:date/set/                              :      0.592s [5589 requests]
+/overview/:date/                                  :      0.467s [19359 requests]
+/clients/:id/                                     :      0.380s [1976 requests]
+........
+
+Top 10 worst DB offenders - cumulative time
+========================================================================
+/overview/:date/                                  :   8773.993s [19359 requests]
+/overview/                                        :   8394.754s [3985 requests]
+/overview/:date/set/                              :   3307.928s [5589 requests]
+/clients/:id/products/:id/                        :   1425.220s [924 requests]
+/clients/:id/                                     :    535.229s [1976 requests]
+........
+
+Top 10 worst DB offenders - mean time
+========================================================================
+/overview/:id/:id/:id/print/                      :      6.994s [448 requests]
+/overview/                                        :      2.128s [3985 requests]
+/clients/:id/products/:id/                        :      1.602s [924 requests]
+/overview/:date/set/                              :      0.592s [5589 requests]
+/overview/:date/                                  :      0.467s [19359 requests]
+........
+
+Mongrel process blockers (> 1.0 seconds)
+========================================================================
+/overview/:date/                                  :   7494.233s [3144 requests]
+/overview/                                        :   8320.293s [1549 requests]
+/overview/:date/set/                              :   1149.235s [803 requests]
+/overview/:id/:id/:id/print/new/                  :    613.693s [341 requests]
+/clients/:id/products/:id/                        :   1370.693s [313 requests]
+........
+
+Requests graph - per hour
+========================================================================
+         ........
+         7:00 - 2731                 : XXXXXXX
+         8:00 - 6139                 : XXXXXXXXXXXXXXXX
+         9:00 - 7465                 : XXXXXXXXXXXXXXXXXXXX
+        10:00 - 7118                 : XXXXXXXXXXXXXXXXXXX
+        11:00 - 7409                 : XXXXXXXXXXXXXXXXXXX
+        12:00 - 6450                 : XXXXXXXXXXXXXXXXX
+        13:00 - 5377                 : XXXXXXXXXXXXXX
+        14:00 - 6058                 : XXXXXXXXXXXXXXXX
+        15:00 - 4156                 : XXXXXXXXXXX
+        16:00 - 2767                 : XXXXXXX
+        17:00 - 1598                 : XXXX
+        18:00 - 792                  : XX
+        ........
+
+Errors
+========================================================================
+ArgumentError: [237 requests]
+ -> invalid date
+StaleObjectError: [28 requests]
+ -> Attempted to update a stale object
+RuntimeError: [3 requests]
+ -> Cannot destroy rule before it was created
+StatementError: [2 requests]
+ -> Mysql::Error: Deadlock found when trying to get lock; try restarting transaction
+NoMethodError: [1 requests]
+ -> undefined method `code' for nil:NilClass

data/Rakefile

@@ -0,0 +1,72 @@
+require 'rubygems'
+
+load 'test/tasks.rake'
+ 
+desc 'Default: run unit tests for request-log-analyzer.'
+task :default => :test
+
+
+namespace :gem do
+
+  desc "Sets the version and date of the gem. Requires the VERSION environment variable."
+  task :version => [:manifest] do
+    
+    require 'date'
+    
+    new_version = ENV['VERSION']
+    raise "VERSION is required" unless /\d+(\.\d+)*/ =~ new_version
+    
+    spec_file = Dir['*.gemspec'].first
+    
+    spec = File.read(spec_file)
+    spec.gsub!(/^(\s*s\.version\s*=\s*)('|")(.+)('|")(\s*)$/) { "#{$1}'#{new_version}'#{$5}" }
+    spec.gsub!(/^(\s*s\.date\s*=\s*)('|")(.+)('|")(\s*)$/) { "#{$1}'#{Date.today.strftime('%Y-%m-%d')}'#{$5}" }    
+    File.open(spec_file, 'w') { |f| f << spec }
+  end
+  
+  desc "Creates a git tag for the provided VERSION"
+  task :tag => [:version] do
+    
+    new_version = ENV['VERSION']
+    raise "VERSION is required" unless /\d+(\.\d+)*/ =~ new_version
+        
+    sh "git add request-log-analyzer.gemspec"
+    sh "git commit -m \"Set gem version to #{new_version}\""
+    sh "git push origin"    
+    sh "git tag -a \"request-log-analyzer-#{new_version}\" -m \"Tagged version #{new_version}\""
+    sh "git push --tags"
+  end
+
+  desc "Builds a ruby gem for request-log-analyzer"
+  task :build => [:manifest] do
+    system %[gem build request-log-analyzer.gemspec]
+  end
+
+  desc %{Update ".manifest" with the latest list of project filenames. Respect\
+  .gitignore by excluding everything that git ignores. Update `files` and\
+  `test_files` arrays in "*.gemspec" file if it's present.}
+  task :manifest do
+    list = Dir['**/*'].sort
+    spec_file = Dir['*.gemspec'].first
+    list -= [spec_file] if spec_file
+  
+    File.read('.gitignore').each_line do |glob|
+      glob = glob.chomp.sub(/^\//, '')
+      list -= Dir[glob]
+      list -= Dir["#{glob}/**/*"] if File.directory?(glob) and !File.symlink?(glob)
+      puts "excluding #{glob}"
+    end
+ 
+    if spec_file
+      spec = File.read spec_file
+      spec.gsub! /^(\s* s.(test_)?files \s* = \s* )( \[ [^\]]* \] | %w\( [^)]* \) )/mx do
+        assignment = $1
+        bunch = $2 ? list.grep(/^test.*_test\.rb$/) : list
+        '%s%%w(%s)' % [assignment, bunch.join(' ')]
+      end
+      
+      File.open(spec_file,   'w') {|f| f << spec }
+    end
+    File.open('.manifest', 'w') {|f| f << list.join("\n") }
+  end
+end

data/TODO

@@ -0,0 +1,18 @@
+TODO items for Rails-log-analyzer
+=================================
+Contact willem AT vanbergen DOT org if you want to help out with the development.
+
+Database:
+- Add query functionality for the resulting database file (interactive reports?)
+- Link request processing line to request completed line
+
+Rails integration:
+- Create script that calls request-log-analyzer 
+- Optionally use local or specific routes.rb file to parse URLs
+- Add rake tasks to Rails application when included
+
+General:
+- Add useful rake tasks
+- Add more tests
+- World domination
+- Fix multiple file handling

data/bin/request-log-analyzer

@@ -0,0 +1,115 @@
+#!/usr/bin/ruby
+require File.dirname(__FILE__) + '/../lib/command_line/arguments'
+require File.dirname(__FILE__) + '/../lib/base/log_parser'
+require File.dirname(__FILE__) + '/../lib/base/summarizer'
+require File.dirname(__FILE__) + '/../lib/rails_analyzer/log_parser'
+require File.dirname(__FILE__) + '/../lib/rails_analyzer/summarizer'
+require File.dirname(__FILE__) + '/../lib/merb_analyzer/log_parser'
+require File.dirname(__FILE__) + '/../lib/merb_analyzer/summarizer'
+require File.dirname(__FILE__) + '/../lib/bashcolorizer'
+require File.dirname(__FILE__) + '/../lib/ruby-progressbar/progressbar.rb'
+
+puts "Request log analyzer, by Willem van Bergen and Bart ten Brinke\n\n"
+
+# Substitutes variable elements in a url (like the id field) with a fixed string (like ":id")
+# This is used to aggregate simular requests. 
+# <tt>request</tt> The request to evaluate.
+# Returns uniformed url string.
+# Raises on mailformed request.
+def request_hasher(request)
+  if request[:url]
+    url = request[:url].downcase.split(/^http[s]?:\/\/[A-z0-9\.-]+/).last.split('?').first # only the relevant URL part
+    url << '/' if url[-1] != '/'[0] && url.length > 1 # pad a trailing slash for consistency
+
+    url.gsub!(/\/\d+-\d+-\d+(\/|$)/, '/:date') # Combine all (year-month-day) queries
+    url.gsub!(/\/\d+-\d+(\/|$)/, '/:month') # Combine all date (year-month) queries
+    url.gsub!(/\/\d+[\w-]*/, '/:id') # replace identifiers in URLs
+        
+    return url
+  elsif request[:controller] && request[:action]
+    return "#{request[:controller]}##{request[:action]}"
+  else
+    raise 'Cannot hash this request! ' + request.inspect
+  end
+end
+
+# Print results using a ASCII table.
+# <tt>summarizer</tt> The summarizer containg information to draw the table.
+# <tt>field</tt> The field containing the data to be printed
+# <tt>amount</tt> The length of the table (defaults to 20)
+def print_table(summarizer, field, amount = 20)
+  summarizer.sort_actions_by(field).reverse[0, amount.to_i].each do |a|
+    # As we show count by default, show totaltime if we sort by count
+    field = :total_time if field == :count
+    
+    puts "%-50s: %10.03fs [#{green("%d requests")}]" % [a[0], a[1][field], a[1][:count]]
+  end
+end
+
+# Parse the arguments given via commandline
+begin
+  $arguments = CommandLine::Arguments.parse do |command_line|
+    command_line.switch(:guess_database_time, :g)
+    command_line.switch(:fast, :f)
+    command_line.switch(:colorize, :z)
+    command_line.switch(:merb, :m)
+    command_line.flag(:output, :alias => :o)
+    command_line.flag(:amount, :alias => :c)
+    command_line.required_files = 1
+  end
+  
+rescue CommandLine::Error => e
+  puts "ARGUMENT ERROR: " + e.message
+  puts
+  load File.dirname(__FILE__) + "/../output/usage.rb"
+  exit(0) 
+end
+
+
+if $arguments[:merb]
+  $summarizer = MerbAnalyzer::Summarizer.new(:calculate_database => $arguments[:guess_database_time])
+else
+  $summarizer = RailsAnalyzer::Summarizer.new(:calculate_database => $arguments[:guess_database_time])
+end
+
+if $arguments[:fast]
+  line_types = [:completed]
+elsif $arguments[:merb]
+  line_types = MerbAnalyzer::LogParser::LOG_LINES.keys
+else
+  line_types = RailsAnalyzer::LogParser::LOG_LINES.keys
+end
+
+# Walk through al the files given via the arguments.
+$arguments.files.each do |log_file|
+  puts "Processing #{line_types.join(', ')} log lines from #{log_file}..."
+
+  if $arguments[:merb]
+    parser = MerbAnalyzer::LogParser.new(log_file)
+  else
+    parser = RailsAnalyzer::LogParser.new(log_file)
+  end
+  
+  # add progress bar 
+  unless $arguments[:fast]
+    pbar = ProgressBar.new(green(log_file), File.size(log_file))
+    parser.progress { |pos, total| (pos == :finished) ? pbar.finish : pbar.set(pos) }
+  end
+  
+  parser.each(*line_types) do |request|
+    $summarizer.group(request)  { |r| request_hasher(r) }
+  end
+end
+
+# Select the reports to output and generate them.
+output_reports = $arguments[:output].split(',') rescue [:timespan, :most_requested, :total_time, :mean_time, :total_db_time, :mean_db_time, :mean_rendering_time, :blockers, :hourly_spread, :errors] 
+
+output_reports.each do |report|
+  report_location = "#{File.dirname(__FILE__)}/../output/#{report}.rb"
+
+  if File.exist?(report_location)
+    load report_location
+  else
+    puts "\nERROR: Output report #{report} not found!"
+  end
+end

data/bin/request-log-database

@@ -0,0 +1,81 @@
+#!/usr/bin/ruby
+
+require File.dirname(__FILE__) + '/../lib/command_line/arguments'
+require File.dirname(__FILE__) + '/../lib/base/log_parser'
+require File.dirname(__FILE__) + '/../lib/base/record_inserter'
+require File.dirname(__FILE__) + '/../lib/rails_analyzer/log_parser'
+require File.dirname(__FILE__) + '/../lib/rails_analyzer/record_inserter'
+require File.dirname(__FILE__) + '/../lib/bashcolorizer'
+require File.dirname(__FILE__) + '/../lib/ruby-progressbar/progressbar.rb'
+
+
+puts "Rails log parser, by Willem van Bergen and Bart ten Brinke\n\n"
+
+begin
+  
+  $arguments = CommandLine::Arguments.parse do |command_line|
+    command_line.switch(:guess_database_time, :g)
+    command_line.switch(:reset_database, :r)
+    command_line.flag(:database, :alias => :d, :required => false)
+    command_line.required_files = 1
+  end
+  
+rescue CommandLine::Error => e
+  puts "ARGUMENT ERROR: " + e.message
+  puts
+  puts "Usage: ruby parsetodb.rb [LOGFILES*] <OPTIONS>"
+  puts
+  puts "Options:"
+  puts "  --database, -t:             The database file to use"
+  puts "  --reset-database, -r:       Resets the database before inserting new records"
+  puts "  --guess-database-time, -g:  Guesses the database duration of requests"      
+  puts
+  puts "Examples:"
+  puts "  ./parsetodb.rb development.log"
+  puts "  ./parsetodb.rb mongrel.0.log mongrel.1.log mongrel.2.log -g -d mongrel.db"  
+  puts 
+   
+  exit(0) 
+end
+
+log_files  = $arguments.files
+db_file    = $arguments[:database] || log_files.first + '.db'
+
+if $arguments[:reset_database] && File.exist?(db_file) 
+  File.delete(db_file) 
+  puts "Database file cleared."
+end
+
+records_inserted = 0 
+inserter = RailsAnalyzer::RecordInserter.insert_batch_into(db_file) do |db|
+  log_files.each do |log_file|
+
+    puts "Processing all log lines from #{log_file}..."
+    parser = RailsAnalyzer::LogParser.new(log_file)
+    
+    pbar = ProgressBar.new(green(log_file), File.size(log_file))
+    parser.progress { |pos, total| (pos == :finished) ? pbar.finish : pbar.set(pos) }
+    
+    parser.each do |request| 
+      db.insert(request) 
+      records_inserted += 1
+    end
+  end
+  
+  if $arguments[:guess_database_time]
+    puts "Calculating database times..."
+    db.calculate_db_durations! 
+  end
+end
+
+started   = inserter.count(:started)
+completed = inserter.count(:completed)
+failed    = inserter.count(:failed)
+
+puts 
+puts "Inserted #{records_inserted} records from #{log_files.length} files."
+puts "Parse warnings: #{inserter.warning_count}. Check the parse_warnings table in the database for details."
+puts
+puts "Requests started: #{started}"
+puts "Requests completed: #{completed}"
+puts "Requests failed: #{failed}"

data/lib/base/log_parser.rb

@@ -0,0 +1,68 @@
+module Base
+  # Parse a log file
+  class LogParser 
+
+    LOG_LINES = {}
+  
+    # LogParser initializer
+    # <tt>file</tt> The fileobject this LogParser wil operate on.
+    def initialize(file, options = {})
+      @file_name = file
+      @options = options
+      @file_size = File.size(@file_name)
+      
+      self.initialize_hook(options) if self.respond_to?(:initialize_hook)
+    end
+  
+    def progress(&block)
+      @progress_handler = block
+    end
+
+    # Output a warning
+    # <tt>message</tt> The warning message (object)
+    def warn(message)
+      puts " -> " + message.to_s
+    end  
+
+    # Finds a log line and then parses the information in the line.
+    # Yields a hash containing the information found. 
+    # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
+    # Yeilds a Hash when it encounters a chunk of information.
+    def each(*line_types, &block)
+      log_lines_hash = self.class::LOG_LINES
+
+      
+      # parse everything by default 
+      line_types = log_lines_hash.keys if line_types.empty?
+
+      File.open(@file_name) do |file|
+      
+        file.each_line do |line|
+        
+          #@progress_handler.call(file.pos, @file_size) if @progress_handler
+          
+          line_types.each do |line_type|
+            if log_lines_hash[line_type][:teaser] =~ line
+              if log_lines_hash[line_type][:regexp] =~ line
+                request = { :type => line_type, :line => file.lineno }
+                log_lines_hash[line_type][:params].each do |key, value|
+                  request[key] = case value
+                    when Numeric; $~[value]
+                    when Array;   $~[value.first].send(value.last)
+                    else; nil
+                  end
+                
+                end
+                
+                yield(request) if block_given?
+              else
+                warn("Unparsable #{line_type} line: " + line[0..79]) unless line_type == :failed
+              end
+            end            
+          end
+        end
+        @progress_handler.call(:finished, @file_size) if @progress_handler
+      end      
+    end
+  end
+end