brontes3d-production_log_analyzer 2009022403

data/lib/production_log/analyzer.rb

@@ -0,0 +1,416 @@
+$TESTING = false unless defined? $TESTING
+
+require 'production_log/parser'
+
+module Enumerable
+
+  ##
+  # Sum of all the elements of the Enumerable
+
+  def sum
+    return self.inject(0) { |acc, i| acc + i }
+  end
+
+  ##
+  # Average of all the elements of the Enumerable
+  #
+  # The Enumerable must respond to #length
+
+  def average
+    return self.sum / self.length.to_f
+  end
+
+  ##
+  # Sample variance of all the elements of the Enumerable
+  #
+  # The Enumerable must respond to #length
+
+  def sample_variance
+    avg = self.average
+    sum = self.inject(0) { |acc, i| acc + (i - avg) ** 2 }
+    return (1 / self.length.to_f * sum)
+  end
+
+  ##
+  # Standard deviation of all the elements of the Enumerable
+  #
+  # The Enumerable must respond to #length
+
+  def standard_deviation
+    return Math.sqrt(self.sample_variance)
+  end
+
+end
+
+##
+# A list that only stores +limit+ items.
+
+class SizedList < Array
+
+  ##
+  # Creates a new SizedList that can hold up to +limit+ items.  Whenever
+  # adding a new item to the SizedList would make the list larger than
+  # +limit+, +delete_block+ is called.
+  #
+  # +delete_block+ is passed the list and the item being added.
+  # +delete_block+ must take action to remove an item and return true or
+  # return false if the item should not be added to the list.
+
+  def initialize(limit, &delete_block)
+    @limit = limit
+    @delete_block = delete_block
+  end
+
+  ##
+  # Attempts to add +obj+ to the list.
+
+  def <<(obj)
+    return super if self.length < @limit
+    return super if @delete_block.call self, obj
+  end
+
+end
+
+##
+# Stores +limit+ time/object pairs, keeping only the largest +limit+ items.
+#
+# Sample usage:
+#
+#   l = SlowestTimes.new 5
+#   
+#   l << [Time.now, 'one']
+#   l << [Time.now, 'two']
+#   l << [Time.now, 'three']
+#   l << [Time.now, 'four']
+#   l << [Time.now, 'five']
+#   l << [Time.now, 'six']
+#
+#   p l.map { |i| i.last }
+
+class SlowestTimes < SizedList
+
+  ##
+  # Creates a new SlowestTimes SizedList that holds only +limit+ time/object
+  # pairs.
+
+  def initialize(limit)
+    super limit do |arr, new_item|
+      fastest_time = arr.sort_by { |time, name| time }.first
+      if fastest_time.first < new_item.first then
+        arr.delete_at index(fastest_time)
+        true
+      else
+        false
+      end
+    end
+  end
+
+end
+
+##
+# Calculates statistics for production logs.
+
+class Analyzer
+
+  ##
+  # The version of the production log analyzer you are using.
+
+  VERSION = '1.5.0'
+
+  ##
+  # The logfile being read by the Analyzer.
+
+  attr_reader :logfile_name
+
+  ##
+  # An Array of all the request total times for the log file.
+
+  attr_reader :request_times
+
+  ##
+  # An Array of all the request database times for the log file.
+
+  attr_reader :db_times
+
+  ##
+  # An Array of all the request render times for the log file.
+
+  attr_reader :render_times
+
+  ##
+  # Generates and sends an email report with lots of fun stuff in it.  This
+  # way, Mail.app will behave when given tabs.
+
+  def self.email(file_name, recipient, subject, count = 10)
+    analyzer = self.new file_name
+    analyzer.process
+    body = analyzer.report count
+
+    email = self.envelope(recipient, subject)
+    email << nil
+    email << "<pre>#{body}</pre>"
+    email = email.join($/) << $/
+
+    return email if $TESTING
+
+    IO.popen("/usr/sbin/sendmail -i -t", "w+") do |sm|
+      sm.print email
+      sm.flush
+    end
+  end
+
+  def self.envelope(recipient, subject = nil) # :nodoc:
+    envelope = {}
+    envelope['To'] = recipient
+    envelope['Subject'] = subject || "pl_analyze"
+    envelope['Content-Type'] = "text/html"
+
+    return envelope.map { |(k,v)| "#{k}: #{v}" }
+  end
+
+  ##
+  # Creates a new Analyzer that will read data from +logfile_name+.
+
+  def initialize(logfile_name)
+    @logfile_name  = logfile_name
+    @request_times = Hash.new { |h,k| h[k] = [] }
+    @db_times      = Hash.new { |h,k| h[k] = [] }
+    @render_times  = Hash.new { |h,k| h[k] = [] }
+  end
+
+  ##
+  # Processes the log file collecting statistics from each found LogEntry.
+
+  def process
+    if File.directory?(@logfile_name)
+      dir_path = @logfile_name
+      Dir.new(dir_path).each do |filename|
+        unless filename[0,1] == "."
+          file_path = File.join(dir_path, filename)
+          File.open file_path do |fp|
+            LogParser.parse fp do |entry|
+              entry_page = entry.page
+              next if entry_page.nil?
+              @request_times[entry_page] << entry.request_time
+              @db_times[entry_page] << entry.db_time
+              @render_times[entry_page] << entry.render_time
+            end
+          end          
+        end
+      end
+    else
+      File.open @logfile_name do |fp|
+        LogParser.parse fp do |entry|
+          entry_page = entry.page
+          next if entry_page.nil?
+          @request_times[entry_page] << entry.request_time
+          @db_times[entry_page] << entry.db_time
+          @render_times[entry_page] << entry.render_time
+        end
+      end
+    end
+  end
+
+  ##
+  # The average total request time for all requests.
+
+  def average_request_time
+    return time_average(@request_times)
+  end
+
+  ##
+  # The standard deviation of the total request time for all requests.
+
+  def request_time_std_dev
+    return time_std_dev(@request_times)
+  end
+
+  ##
+  # The +limit+ slowest total request times.
+
+  def slowest_request_times(limit = 10)
+    return slowest_times(@request_times, limit)
+  end
+
+  ##
+  # The average total database time for all requests.
+
+  def average_db_time
+    return time_average(@db_times)
+  end
+
+  ##
+  # The standard deviation of the total database time for all requests.
+
+  def db_time_std_dev
+    return time_std_dev(@db_times)
+  end
+
+  ##
+  # The +limit+ slowest total database times.
+
+  def slowest_db_times(limit = 10)
+    return slowest_times(@db_times, limit)
+  end
+
+  ##
+  # The average total render time for all requests.
+
+  def average_render_time
+    return time_average(@render_times)
+  end
+
+  ##
+  # The standard deviation of the total render time for all requests.
+
+  def render_time_std_dev
+    return time_std_dev(@render_times)
+  end
+
+  ##
+  # The +limit+ slowest total render times for all requests.
+
+  def slowest_render_times(limit = 10)
+    return slowest_times(@render_times, limit)
+  end
+
+  ##
+  # A list of count/min/max/avg/std dev for request times.
+
+  def request_times_summary
+    return summarize("Request Times", @request_times)
+  end
+
+  ##
+  # A list of count/min/max/avg/std dev for database times.
+
+  def db_times_summary
+    return summarize("DB Times", @db_times)
+  end
+
+  ##
+  # A list of count/min/max/avg/std dev for request times.
+
+  def render_times_summary
+    return summarize("Render Times", @render_times)
+  end
+
+  ##
+  # Builds a report containing +count+ slow items.
+
+  def report(count)
+    return "No requests to analyze" if request_times.empty?
+
+    text = []
+
+    text << request_times_summary
+    text << nil
+    text << "Slowest Request Times:"
+    slowest_request_times(count).each do |time, name|
+      text << "\t#{name} took #{'%0.3f' % time}s"
+    end
+    text << nil
+    text << "-" * 72
+    text << nil
+
+    text << db_times_summary
+    text << nil
+    text << "Slowest Total DB Times:"
+    slowest_db_times(count).each do |time, name|
+      text << "\t#{name} took #{'%0.3f' % time}s"
+    end
+    text << nil
+    text << "-" * 72
+    text << nil
+
+    text << render_times_summary
+    text << nil
+    text << "Slowest Total Render Times:"
+    slowest_render_times(count).each do |time, name|
+      unless ('%0.3f' % time) == "0.000"
+        text << "\t#{name} took #{'%0.3f' % time}s"
+      end
+    end
+    text << nil
+
+    return text.join($/)
+  end
+
+  private unless $TESTING
+
+  def summarize(title, records) # :nodoc:
+    record = nil
+    list = []
+
+    # header
+    record = [pad_request_name("#{title} Summary"), 'Count', 'Avg', 'Std Dev',
+              'Min', 'Max']
+    list << record.join("\t")
+
+    # all requests
+    times = records.values.flatten
+    record = [times.average, times.standard_deviation, times.min, times.max]
+    record.map! { |v| "%0.3f" % v }
+    record.unshift [pad_request_name('ALL REQUESTS'), times.size]
+    list << record.join("\t")
+
+    # spacer
+    list << nil
+
+    records.sort_by { |k,v| v.size}.reverse_each do |req, times|
+      record = [times.average, times.standard_deviation, times.min, times.max]
+      record.map! { |v| "%0.3f" % v }
+      record.unshift ["#{pad_request_name req}", times.size]
+      list << record.join("\t")
+    end
+
+    return list.join("\n")
+  end
+
+  def slowest_times(records, limit) # :nodoc:
+    slowest_times = SlowestTimes.new limit
+
+    records.each do |name, times|
+      times.each do |time|
+        slowest_times << [time, name]
+      end
+    end
+
+    return slowest_times.sort_by { |time, name| time }.reverse
+  end
+
+  def time_average(records) # :nodoc:
+    times = records.values.flatten
+    times.delete 0
+    return times.average
+  end
+
+  def time_std_dev(records) # :nodoc:
+    times = records.values.flatten
+    times.delete 0
+    return times.standard_deviation
+  end
+
+  def longest_request_name # :nodoc:
+    return @longest_req if defined? @longest_req
+
+    names = @request_times.keys.map do |name|
+      (name||'Unknown').length + 1 # + : - HACK where does nil come from?
+    end
+
+    @longest_req = names.max
+
+    @longest_req = 'Unknown'.length + 1 if @longest_req.nil?
+
+    return @longest_req
+  end
+
+  def pad_request_name(name) # :nodoc:
+    name = (name||'Unknown') + ':' # HACK where does nil come from?
+    padding_width = longest_request_name - name.length
+    padding_width = 0 if padding_width < 0
+    name += (' ' * padding_width)
+  end
+
+end
+

data/lib/production_log/parser.rb

@@ -0,0 +1,228 @@
+##
+# LogParser parses a Syslog log file looking for lines logged by the 'rails'
+# program.  A typical log line looks like this:
+#
+#   Mar  7 00:00:20 online1 rails[59600]: Person Load (0.001884)   SELECT * FROM people WHERE id = 10519 LIMIT 1
+#
+# LogParser does not work with Rails' default logger because there is no way
+# to group all the log output of a single request.  You must use SyslogLogger.
+
+module LogParser
+  
+  def self.syslog_mode?
+    @syslog_mode
+  end
+  
+  def self.syslog_mode!
+    @syslog_mode = true
+  end
+  def self.vanilla_mode!
+    @syslog_mode = false
+  end
+  
+  ##
+  # LogEntry contains a summary of log data for a single request.
+
+  class LogEntry
+
+    ##
+    # Controller and action for this request
+
+    attr_reader :page
+
+    ##
+    # Requesting IP
+
+    attr_reader :ip
+
+    ##
+    # Time the request was made
+
+    attr_reader :time
+
+    ##
+    # Array of SQL queries containing query type and time taken.  The
+    # complete text of the SQL query is not saved to reduct memory usage.
+
+    attr_reader :queries
+
+    ##
+    # Total request time, including database, render and other.
+
+    attr_reader :request_time
+
+    ##
+    # Total render time.
+
+    attr_reader :render_time
+
+    ##
+    # Total database time
+
+    attr_reader :db_time
+
+    ##
+    # Creates a new LogEntry from the log data in +entry+.
+
+    attr_reader :row_count, :query_count, :request_size, :response_size
+
+    def initialize(entry)
+      @page = nil
+      @ip = nil
+      @time = nil
+      @queries = []
+      @request_time = 0
+      @render_time = 0
+      @db_time = 0
+      @in_component = 0
+
+      parse entry
+    end
+
+    ##
+    # Extracts log data from +entry+, which is an Array of lines from the
+    # same request.
+
+    def parse(entry)
+      entry.each do |line|
+        case line
+        when /^Parameters/, /^Cookie set/, /^Rendering/,
+          /^Redirected/ then
+          # nothing
+        when /^Processing ([\S]+) \(for (.+) at (.*)\)/ then
+          next if @in_component > 0
+          @page = $1
+          @ip   = $2
+          @time = $3
+        when /^Completed in ([\S]+) \(\d* reqs\/sec\) \| (.+)/,
+          /^Completed in ([\S]+) \((.+)\)/ then
+
+          next if @in_component > 0
+          # handle millisecond times as well as fractional seconds
+          @times_in_milliseconds = $1[-2..-1] == 'ms'
+
+          @request_time = @times_in_milliseconds ? ($1.to_i/1000.0) : $1.to_f
+          log_info = $2
+
+          log_info = log_info.split(/[,|]/)
+          log_info = log_info.map do |entry|
+            next nil unless entry.index(': ')
+            result = entry.strip.split(': ')
+            if result.size > 2
+              result = [result[0], result[1..-1].join(':')]
+            end
+            result
+          end.compact.flatten
+
+          log_info = Hash[*log_info]
+
+          @row_count = log_info['Rows'].to_i
+          @query_count = log_info['Queries'].to_i
+          @request_size = log_info['Request Size'].to_i
+          @response_size = log_info['Response Size'].to_i
+
+          @page = log_info['Processed'] if log_info['Processed']
+          @page += ".#{log_info['Response Format']}" if log_info['Response Format']
+
+          if x = (log_info['DB'])
+            x = x.split(' ').first
+            @db_time = @times_in_milliseconds ? (x.to_i/1000.0) : x.to_f
+          end
+
+          if x = (log_info['Rendering'] || log_info['View'])
+            x = x.split(' ').first
+            @render_time = @times_in_milliseconds ? (x.to_i/1000.0) : x.to_f
+          end
+
+        when /(.+?) \(([^)]+)\)   / then
+          @queries << [$1, $2.to_f]
+        when /^Start rendering component / then
+          @in_component += 1
+        when /^End of component rendering$/ then
+          @in_component -= 1
+        when /^Fragment hit: / then
+        else # noop
+#          raise "Can't handle #{line.inspect}" if $TESTING
+        end
+      end
+    end
+
+    def ==(other) # :nodoc:
+      other.class == self.class and
+      other.page == self.page and
+      other.ip == self.ip and
+      other.time == self.time and
+      other.queries == self.queries and
+      other.request_time == self.request_time and
+      other.render_time == self.render_time and
+      other.db_time == self.db_time
+    end
+
+  end
+
+  def self.extract_bucket_and_data(line)
+    if LogParser.syslog_mode?
+      line =~ / ([^ ]+) ([^ ]+)\[(\d+)\]: (.*)/
+      return nil if $2.nil? or $2 == 'newsyslog'
+      bucket = [$1, $2, $3].join '-'
+      data = $4
+      return [bucket, data]
+    else
+      return ["(none)", line]
+    end    
+  end
+
+  def self.detect_mode(stream)
+    lines_read = []
+    while(!stream.eof? && lines_read.size < 5)
+      lines_read << stream.readline
+    end
+    stream.rewind
+    
+    lines_read.each do |line|
+      line =~ / ([^ ]+) ([^ ]+)\[(\d+)\]: (.*)/
+      if $4        
+        return syslog_mode!
+      end
+    end
+    return vanilla_mode!
+  end
+
+  ##
+  # Parses IO stream +stream+, creating a LogEntry for each recognizable log
+  # entry.
+  #
+  # Log entries are recognised as starting with Processing, continuing with
+  # the same process id through Completed.
+
+  def self.parse(stream) # :yields: log_entry
+    buckets = Hash.new { |h,k| h[k] = [] }
+    comp_count = Hash.new 0
+    
+    LogParser.detect_mode(stream)
+    
+    stream.each_line do |line|
+      bucket, data = LogParser.extract_bucket_and_data(line)
+      next if !bucket
+      
+      buckets[bucket] << data
+
+      case data
+      when /^Start rendering component / then
+        comp_count[bucket] += 1
+      when /^End of component rendering$/ then
+        comp_count[bucket] -= 1
+      when /^Completed/ then
+        next unless comp_count[bucket] == 0
+        entry = buckets.delete bucket
+        yield LogEntry.new(entry)
+      end
+    end
+
+    buckets.each do |bucket, data|
+      yield LogEntry.new(data)
+    end
+  end
+
+end
+