erkki-production_log_analyzer 2009022401

data/History.txt

@@ -0,0 +1,34 @@
+= 1.5.0
+
+* Fixed empty log bug.  Patch by Tim Lucas.
+* Fixed bug where sometimes lines would be logged before the
+  Processing line.  Patch by Geoff Grosenbach.
+
+= 1.4.0
+
+* Switched to Hoe
+* Allowed action_errors to suppress routing errors with > 3 occurances
+* action_grep now works correctly with components
+* pl_analyze now works correctly with components
+* Added action_errors to extract error counts from logs
+* Retabbed to match the rest of the world
+
+= 1.3.0
+
+* Added action_grep
+* Added support for newer log format
+
+= 1.2.0
+
+* pl_analyze calculates per-action statistics
+* pl_analyze can send an email with its output
+
+= 1.1.0
+
+* RDoc
+* Other various fixes lost to time.
+
+= 1.0.0
+
+* Birthday!
+

data/LICENSE.txt

@@ -0,0 +1,27 @@
+Copyright 2005, 2007 Eric Hodel, The Robot Co-op.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the names of the authors nor the names of their contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

data/Manifest.txt

@@ -0,0 +1,18 @@
+History.txt
+LICENSE.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/action_errors
+bin/action_grep
+bin/pl_analyze
+lib/production_log/action_grep.rb
+lib/production_log/analyzer.rb
+lib/production_log/parser.rb
+test/test.syslog.0.14.x.log
+test/test.syslog.1.2.shortname.log
+test/test.syslog.empty.log
+test/test.syslog.log
+test/test_action_grep.rb
+test/test_analyzer.rb
+test/test_parser.rb

data/README.txt

@@ -0,0 +1,147 @@
+= production_log_analyzer
+
+production_log_analyzer lets you find out which actions on a Rails
+site are slowing you down.
+
+http://seattlerb.rubyforge.org/production_log_analyzer
+
+http://rubyforge.org/projects/seattlerb
+
+Bug reports:
+
+http://rubyforge.org/tracker/?func=add&group_id=1513&atid=5921
+
+== About
+
+production_log_analyzer provides three tools to analyze log files
+created by SyslogLogger.  pl_analyze for getting daily reports,
+action_grep for pulling log lines for a single action and
+action_errors to summarize errors with counts.
+
+The analyzer currently requires the use of SyslogLogger because the
+default Logger doesn't give any way to associate lines logged to a
+request.
+
+The PL Analyzer also includes action_grep which lets you grab lines from a log
+that only match a single action.
+
+  action_grep RssController#uber /var/log/production.log
+
+== Installing
+
+  sudo gem install production_log_analyzer
+
+=== Setup
+
+First:
+
+Set up SyslogLogger according to the instructions here:
+
+http://seattlerb.rubyforge.org/SyslogLogger/
+
+Then:
+
+Set up a cronjob (or something like that) to run log files through pl_analyze.
+
+== Using pl_analyze
+
+To run pl_analyze simply give it the name of a log file to analyze.
+
+  pl_analyze /var/log/production.log
+
+If you want, you can run it from a cron something like this:
+
+  /usr/bin/gzip -dc /var/log/production.log.0.gz | /usr/local/bin/pl_analyze /dev/stdin
+
+Or, have pl_analyze email you (which is preferred, because tabs get preserved):
+
+  /usr/bin/gzip -dc /var/log/production.log.0.gz | /usr/local/bin/pl_analyze /dev/stdin -e devnull@robotcoop.com -s "pl_analyze for `date -v-1d "+%D"`"
+
+In the future, pl_analyze will be able to read from STDIN.
+
+== Sample output
+
+  Request Times Summary:          Count   Avg     Std Dev Min     Max
+  ALL REQUESTS:                   11      0.576   0.508   0.000   1.470
+  
+  ThingsController#view:          3       0.716   0.387   0.396   1.260
+  TeamsController#progress:       2       0.841   0.629   0.212   1.470
+  RssController#uber:             2       0.035   0.000   0.035   0.035
+  PeopleController#progress:      2       0.489   0.489   0.000   0.977
+  PeopleController#view:          2       0.731   0.371   0.360   1.102
+  
+  Average Request Time: 0.634
+  Request Time Std Dev: 0.498
+  
+  Slowest Request Times:
+          TeamsController#progress took 1.470s
+          ThingsController#view took 1.260s
+          PeopleController#view took 1.102s
+          PeopleController#progress took 0.977s
+          ThingsController#view took 0.492s
+          ThingsController#view took 0.396s
+          PeopleController#view took 0.360s
+          TeamsController#progress took 0.212s
+          RssController#uber took 0.035s
+          RssController#uber took 0.035s
+  
+  ------------------------------------------------------------------------
+  
+  DB Times Summary:               Count   Avg     Std Dev Min     Max
+  ALL REQUESTS:                   11      0.366   0.393   0.000   1.144
+  
+  ThingsController#view:          3       0.403   0.362   0.122   0.914
+  TeamsController#progress:       2       0.646   0.497   0.149   1.144
+  RssController#uber:             2       0.008   0.000   0.008   0.008
+  PeopleController#progress:      2       0.415   0.415   0.000   0.830
+  PeopleController#view:          2       0.338   0.149   0.189   0.486
+  
+  Average DB Time: 0.402
+  DB Time Std Dev: 0.394
+  
+  Slowest Total DB Times:
+          TeamsController#progress took 1.144s
+          ThingsController#view took 0.914s
+          PeopleController#progress took 0.830s
+          PeopleController#view took 0.486s
+          PeopleController#view took 0.189s
+          ThingsController#view took 0.173s
+          TeamsController#progress took 0.149s
+          ThingsController#view took 0.122s
+          RssController#uber took 0.008s
+          RssController#uber took 0.008s
+  
+  ------------------------------------------------------------------------
+  
+  Render Times Summary:           Count   Avg     Std Dev Min     Max
+  ALL REQUESTS:                   11      0.219   0.253   0.000   0.695
+  
+  ThingsController#view:          3       0.270   0.171   0.108   0.506
+  TeamsController#progress:       2       0.000   0.000   0.000   0.000
+  RssController#uber:             2       0.012   0.000   0.012   0.012
+  PeopleController#progress:      2       0.302   0.302   0.000   0.604
+  PeopleController#view:          2       0.487   0.209   0.278   0.695
+  
+  Average Render Time: 0.302
+  Render Time Std Dev: 0.251
+  
+  Slowest Total Render Times:
+          PeopleController#view took 0.695s
+          PeopleController#progress took 0.604s
+          ThingsController#view took 0.506s
+          PeopleController#view took 0.278s
+          ThingsController#view took 0.197s
+          ThingsController#view took 0.108s
+          RssController#uber took 0.012s
+          RssController#uber took 0.012s
+          TeamsController#progress took 0.000s
+          TeamsController#progress took 0.000s
+
+== What's missing
+
+* More reports
+* Command line arguments including:
+  * Help
+  * What type of log file you've got (if somebody sends patches with tests)
+* Read from STDIN
+

data/Rakefile

@@ -0,0 +1,17 @@
+require 'hoe'
+
+$:.unshift './lib'
+require 'production_log/analyzer'
+
+Hoe.new 'production_log_analyzer', '2009022401' do |p|
+  p.summary = p.paragraphs_of('README.txt', 1).join ' '
+  p.description = p.paragraphs_of('README.txt', 7).join ' '
+  p.author = 'Eric Hodel'
+  p.email = 'drbrain@segment7.net'
+  p.url = p.paragraphs_of('README.txt', 2).join ' '
+
+  p.rubyforge_name = 'seattlerb'
+
+  p.extra_deps << ['rails_analyzer_tools', '>= 1.4.0']
+end
+

data/bin/action_errors

@@ -0,0 +1,46 @@
+#!/usr/local/bin/ruby -ws
+
+$h ||= false
+$r ||= false
+$o ||= false
+
+$r = $r ? ($r.to_i rescue false) : false
+
+if $h then
+  $stderr.puts "Usage: #{$0} [-r=N] LOGFILE"
+  $stderr.puts "\t-r=N\tShow routing errors with N or more occurances"
+  $stderr.puts "\t-o\tShow errors with one occurance"
+  exit
+end
+
+errors = {}
+counts = Hash.new 0
+
+ARGF.each_line do |line|
+  line =~ /\]: (.*?)     (.*)/
+  next if $1.nil?
+  msg = $1
+  trace = $2
+  key = msg.gsub(/\d/, '#')
+  counts[key] += 1
+  next if counts[key] > 1
+  trace = trace.split('     ')[0..-2].map { |l| l.strip }.join("\n\t")
+  error = "#{msg}\n\t#{trace}"
+  errors[key] = error
+end
+
+counts.sort_by { |_,c| -c }.each do |key, count|
+  next if count == 1 and not $o
+  error = errors[key]
+
+  if error =~ /^ActionController::RoutingError/ then
+    next unless $r
+    next if $r and count < $r
+  end
+
+  puts "count: #{count}"
+  puts "{{{"
+  puts error
+  puts "}}}"
+end
+

data/bin/action_grep

@@ -0,0 +1,19 @@
+#!/usr/local/bin/ruby -w
+
+require 'production_log/action_grep'
+
+action_name = ARGV.shift
+file_name = ARGV.shift
+
+if action_name.nil? or file_name.nil? then
+  puts "Usage: #{$0} action_name file_name"
+  exit 1
+end
+
+begin
+  ActionGrep.grep action_name, file_name
+rescue ArgumentError => e
+  puts e
+  exit 1
+end
+

data/bin/pl_analyze

@@ -0,0 +1,36 @@
+#!/bin/env ruby -w
+
+$:.unshift "#{File.dirname(__FILE__)}/../lib/"
+require "production_log/analyzer"
+
+file_name = ARGV.shift
+
+if file_name.nil? then
+  puts "Usage: #{$0} file_name [-e email_recipient [-s subject]] [count]"
+  exit 1
+end
+
+email_recipient = nil
+subject = nil
+
+if ARGV.first == '-e' then
+  ARGV.shift # -e
+  email_recipient = ARGV.shift
+end
+
+if email_recipient and ARGV.first == '-s' then
+  ARGV.shift # -s
+  subject = ARGV.shift
+end
+
+count = ARGV.shift
+count = count.nil? ? 10 : Integer(count)
+
+if email_recipient.nil? then
+  analyzer = Analyzer.new file_name
+  analyzer.process
+  puts analyzer.report(count)
+else
+  Analyzer.email file_name, email_recipient, subject, count
+end
+

data/lib/production_log/action_grep.rb

@@ -0,0 +1,42 @@
+module ActionGrep; end
+
+class << ActionGrep
+
+  def grep(action_name, file_name)
+    unless action_name =~ /\A([A-Z][A-Za-z\d]*)(?:#([A-Za-z]\w*))?\Z/ then
+      raise ArgumentError, "Invalid action name #{action_name} expected something like SomeController#action"
+    end
+
+    unless File.file? file_name and File.readable? file_name then
+      raise ArgumentError, "Unable to read #{file_name}"
+    end
+
+    buckets = Hash.new { |h,k| h[k] = [] }
+    comp_count = Hash.new 0
+
+    File.open file_name do |fp|
+      fp.each_line do |line|
+        line =~ / ([^ ]+) ([^ ]+)\[(\d+)\]: (.*)/
+        next if $2.nil? or $2 == 'newsyslog'
+        bucket = [$1, $2, $3].join '-'
+        data = $4
+
+        buckets[bucket] << line
+
+        case data
+        when /^Start rendering component / then
+          comp_count[bucket] += 1
+        when /^End of component rendering$/ then
+          comp_count[bucket] -= 1
+        when /^Completed/ then
+          next unless comp_count[bucket] == 0
+          action = buckets.delete bucket
+          next unless action.any? { |l| l =~ /: Processing #{action_name}/ }
+          puts action.join
+        end
+      end
+    end
+  end
+
+end
+

data/lib/production_log/analyzer.rb

@@ -0,0 +1,406 @@
+$TESTING = false unless defined? $TESTING
+
+require 'production_log/parser'
+
+module Enumerable
+
+  ##
+  # Sum of all the elements of the Enumerable
+
+  def sum
+    return self.inject(0) { |acc, i| acc + i }
+  end
+
+  ##
+  # Average of all the elements of the Enumerable
+  #
+  # The Enumerable must respond to #length
+
+  def average
+    return self.sum / self.length.to_f
+  end
+
+  ##
+  # Sample variance of all the elements of the Enumerable
+  #
+  # The Enumerable must respond to #length
+
+  def sample_variance
+    avg = self.average
+    sum = self.inject(0) { |acc, i| acc + (i - avg) ** 2 }
+    return (1 / self.length.to_f * sum)
+  end
+
+  ##
+  # Standard deviation of all the elements of the Enumerable
+  #
+  # The Enumerable must respond to #length
+
+  def standard_deviation
+    return Math.sqrt(self.sample_variance)
+  end
+
+end
+
+##
+# A list that only stores +limit+ items.
+
+class SizedList < Array
+
+  ##
+  # Creates a new SizedList that can hold up to +limit+ items.  Whenever
+  # adding a new item to the SizedList would make the list larger than
+  # +limit+, +delete_block+ is called.
+  #
+  # +delete_block+ is passed the list and the item being added.
+  # +delete_block+ must take action to remove an item and return true or
+  # return false if the item should not be added to the list.
+
+  def initialize(limit, &delete_block)
+    @limit = limit
+    @delete_block = delete_block
+  end
+
+  ##
+  # Attempts to add +obj+ to the list.
+
+  def <<(obj)
+    return super if self.length < @limit
+    return super if @delete_block.call self, obj
+  end
+
+end
+
+##
+# Stores +limit+ time/object pairs, keeping only the largest +limit+ items.
+#
+# Sample usage:
+#
+#   l = SlowestTimes.new 5
+#   
+#   l << [Time.now, 'one']
+#   l << [Time.now, 'two']
+#   l << [Time.now, 'three']
+#   l << [Time.now, 'four']
+#   l << [Time.now, 'five']
+#   l << [Time.now, 'six']
+#
+#   p l.map { |i| i.last }
+
+class SlowestTimes < SizedList
+
+  ##
+  # Creates a new SlowestTimes SizedList that holds only +limit+ time/object
+  # pairs.
+
+  def initialize(limit)
+    super limit do |arr, new_item|
+      fastest_time = arr.sort_by { |time, name| time }.first
+      if fastest_time.first < new_item.first then
+        arr.delete_at index(fastest_time)
+        true
+      else
+        false
+      end
+    end
+  end
+
+end
+
+##
+# Calculates statistics for production logs.
+
+class Analyzer
+
+  ##
+  # The version of the production log analyzer you are using.
+
+  VERSION = '1.5.0'
+
+  ##
+  # The logfile being read by the Analyzer.
+
+  attr_reader :logfile_name
+
+  ##
+  # An Array of all the request total times for the log file.
+
+  attr_reader :request_times
+
+  ##
+  # An Array of all the request database times for the log file.
+
+  attr_reader :db_times
+
+  ##
+  # An Array of all the request render times for the log file.
+
+  attr_reader :render_times
+  
+  attr_reader :row_counts, :query_counts
+
+  ##
+  # Generates and sends an email report with lots of fun stuff in it.  This
+  # way, Mail.app will behave when given tabs.
+
+  def self.email(file_name, recipient, subject, count = 10)
+    analyzer = self.new file_name
+    analyzer.process
+    body = analyzer.report count
+
+    email = self.envelope(recipient, subject)
+    email << nil
+    email << "<pre>#{body}</pre>"
+    email = email.join($/) << $/
+
+    return email if $TESTING
+
+    IO.popen("/usr/sbin/sendmail -i -t", "w+") do |sm|
+      sm.print email
+      sm.flush
+    end
+  end
+
+  def self.envelope(recipient, subject = nil) # :nodoc:
+    envelope = {}
+    envelope['To'] = recipient
+    envelope['Subject'] = subject || "pl_analyze"
+    envelope['Content-Type'] = "text/html"
+
+    return envelope.map { |(k,v)| "#{k}: #{v}" }
+  end
+
+  ##
+  # Creates a new Analyzer that will read data from +logfile_name+.
+
+  def initialize(logfile_name)
+    @logfile_name  = logfile_name
+    @request_times = Hash.new { |h,k| h[k] = [] }
+    @db_times      = Hash.new { |h,k| h[k] = [] }
+    @row_counts    = Hash.new { |h,k| h[k] = [] }
+    @query_counts  = Hash.new { |h,k| h[k] = [] }
+    @render_times  = Hash.new { |h,k| h[k] = [] }
+  end
+
+  ##
+  # Processes the log file collecting statistics from each found LogEntry.
+
+  def process
+    File.open @logfile_name do |fp|
+      LogParser.parse fp do |entry|
+        entry_page = entry.page
+        next if entry_page.nil?
+        @request_times[entry_page] << entry.request_time
+        @db_times[entry_page] << entry.db_time
+        @row_counts[entry_page] << entry.row_count
+        @query_counts[entry_page] << entry.query_count
+        @render_times[entry_page] << entry.render_time
+      end
+    end
+  end
+
+  ##
+  # The average total request time for all requests.
+
+  def average_request_time
+    return time_average(@request_times)
+  end
+
+  ##
+  # The standard deviation of the total request time for all requests.
+
+  def request_time_std_dev
+    return time_std_dev(@request_times)
+  end
+
+  ##
+  # The +limit+ slowest total request times.
+
+  def slowest_request_times(limit = 10)
+    return slowest_times(@request_times, limit)
+  end
+
+  ##
+  # The average total database time for all requests.
+
+  def average_db_time
+    return time_average(@db_times)
+  end
+
+  ##
+  # The standard deviation of the total database time for all requests.
+
+  def db_time_std_dev
+    return time_std_dev(@db_times)
+  end
+
+  ##
+  # The +limit+ slowest total database times.
+
+  def slowest_db_times(limit = 10)
+    return slowest_times(@db_times, limit)
+  end
+
+  ##
+  # The average total render time for all requests.
+
+  def average_render_time
+    return time_average(@render_times)
+  end
+
+  ##
+  # The standard deviation of the total render time for all requests.
+
+  def render_time_std_dev
+    return time_std_dev(@render_times)
+  end
+
+  ##
+  # The +limit+ slowest total render times for all requests.
+
+  def slowest_render_times(limit = 10)
+    return slowest_times(@render_times, limit)
+  end
+
+  ##
+  # A list of count/min/max/avg/std dev for request times.
+
+  def request_times_summary
+    return summarize("Request Times", @request_times)
+  end
+
+  ##
+  # A list of count/min/max/avg/std dev for database times.
+
+  def db_times_summary
+    return summarize("DB Times", @db_times)
+  end
+
+  ##
+  # A list of count/min/max/avg/std dev for request times.
+
+  def render_times_summary
+    return summarize("Render Times", @render_times)
+  end
+
+  ##
+  # Builds a report containing +count+ slow items.
+
+  def report(count)
+    return "No requests to analyze" if request_times.empty?
+
+    text = []
+
+    text << request_times_summary
+    text << nil
+    text << "Slowest Request Times:"
+    slowest_request_times(count).each do |time, name|
+      text << "\t#{name} took #{'%0.3f' % time}s"
+    end
+    text << nil
+    text << "-" * 72
+    text << nil
+
+    text << db_times_summary
+    text << nil
+    text << "Slowest Total DB Times:"
+    slowest_db_times(count).each do |time, name|
+      text << "\t#{name} took #{'%0.3f' % time}s"
+    end
+    text << nil
+    text << "-" * 72
+    text << nil
+
+    text << render_times_summary
+    text << nil
+    text << "Slowest Total Render Times:"
+    slowest_render_times(count).each do |time, name|
+      text << "\t#{name} took #{'%0.3f' % time}s"
+    end
+    text << nil
+
+    return text.join($/)
+  end
+
+  private unless $TESTING
+
+  def summarize(title, records) # :nodoc:
+    record = nil
+    list = []
+
+    # header
+    record = [pad_request_name("#{title} Summary"), 'Count', 'Avg', 'Std Dev',
+              'Min', 'Max', 'Queries', 'Rows']
+    list << record.join("\t")
+
+    # all requests
+    times = records.values.flatten
+    record = [times.average, times.standard_deviation, times.min, times.max, 0, 0]
+    record.map! { |v| "%0.3f" % v }
+    record.unshift [pad_request_name('ALL REQUESTS'), times.size]
+    list << record.join("\t")
+
+    # spacer
+    list << nil
+
+    records.sort_by { |k,v| v.size}.reverse_each do |req, times|
+      if "DB Times" == title
+        average_rows = @row_counts[req].average
+        average_queries = @query_counts[req].average
+      end
+      record = [times.average, times.standard_deviation, times.min, times.max, average_queries || 0, average_rows || 0]
+      record.map! { |v| "%0.3f" % v }
+      record.unshift ["#{pad_request_name req}", times.size]
+      list << record.join("\t")
+    end
+
+    return list.join("\n")
+  end
+
+  def slowest_times(records, limit) # :nodoc:
+    slowest_times = SlowestTimes.new limit
+
+    records.each do |name, times|
+      times.each do |time|
+        slowest_times << [time, name]
+      end
+    end
+
+    return slowest_times.sort_by { |time, name| time }.reverse
+  end
+
+  def time_average(records) # :nodoc:
+    times = records.values.flatten
+    times.delete 0
+    return times.average
+  end
+
+  def time_std_dev(records) # :nodoc:
+    times = records.values.flatten
+    times.delete 0
+    return times.standard_deviation
+  end
+
+  def longest_request_name # :nodoc:
+    return @longest_req if defined? @longest_req
+
+    names = @request_times.keys.map do |name|
+      (name||'Unknown').length + 1 # + : - HACK where does nil come from?
+    end
+
+    @longest_req = names.max
+
+    @longest_req = 'Unknown'.length + 1 if @longest_req.nil?
+
+    return @longest_req
+  end
+
+  def pad_request_name(name) # :nodoc:
+    name = (name||'Unknown') + ':' # HACK where does nil come from?
+    padding_width = longest_request_name - name.length
+    padding_width = 0 if padding_width < 0
+    name += (' ' * padding_width)
+  end
+
+end
+