brontes3d-production_log_analyzer 2009022403

data/History.txt

@@ -0,0 +1,34 @@
+= 1.5.0
+
+* Fixed empty log bug.  Patch by Tim Lucas.
+* Fixed bug where sometimes lines would be logged before the
+  Processing line.  Patch by Geoff Grosenbach.
+
+= 1.4.0
+
+* Switched to Hoe
+* Allowed action_errors to suppress routing errors with > 3 occurances
+* action_grep now works correctly with components
+* pl_analyze now works correctly with components
+* Added action_errors to extract error counts from logs
+* Retabbed to match the rest of the world
+
+= 1.3.0
+
+* Added action_grep
+* Added support for newer log format
+
+= 1.2.0
+
+* pl_analyze calculates per-action statistics
+* pl_analyze can send an email with its output
+
+= 1.1.0
+
+* RDoc
+* Other various fixes lost to time.
+
+= 1.0.0
+
+* Birthday!
+

data/LICENSE.txt

@@ -0,0 +1,27 @@
+Copyright 2005, 2007 Eric Hodel, The Robot Co-op.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the names of the authors nor the names of their contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

data/Manifest.txt

@@ -0,0 +1,18 @@
+History.txt
+LICENSE.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/action_errors
+bin/action_grep
+bin/pl_analyze
+lib/production_log/action_grep.rb
+lib/production_log/analyzer.rb
+lib/production_log/parser.rb
+test/test.syslog.0.14.x.log
+test/test.syslog.1.2.shortname.log
+test/test.syslog.empty.log
+test/test.syslog.log
+test/test_action_grep.rb
+test/test_analyzer.rb
+test/test_parser.rb

data/README.txt

@@ -0,0 +1,147 @@
+= production_log_analyzer
+
+production_log_analyzer lets you find out which actions on a Rails
+site are slowing you down.
+
+http://seattlerb.rubyforge.org/production_log_analyzer
+
+http://rubyforge.org/projects/seattlerb
+
+Bug reports:
+
+http://rubyforge.org/tracker/?func=add&group_id=1513&atid=5921
+
+== About
+
+production_log_analyzer provides three tools to analyze log files
+created by SyslogLogger.  pl_analyze for getting daily reports,
+action_grep for pulling log lines for a single action and
+action_errors to summarize errors with counts.
+
+The analyzer currently requires the use of SyslogLogger because the
+default Logger doesn't give any way to associate lines logged to a
+request.
+
+The PL Analyzer also includes action_grep which lets you grab lines from a log
+that only match a single action.
+
+  action_grep RssController#uber /var/log/production.log
+
+== Installing
+
+  sudo gem install production_log_analyzer
+
+=== Setup
+
+First:
+
+Set up SyslogLogger according to the instructions here:
+
+http://seattlerb.rubyforge.org/SyslogLogger/
+
+Then:
+
+Set up a cronjob (or something like that) to run log files through pl_analyze.
+
+== Using pl_analyze
+
+To run pl_analyze simply give it the name of a log file to analyze.
+
+  pl_analyze /var/log/production.log
+
+If you want, you can run it from a cron something like this:
+
+  /usr/bin/gzip -dc /var/log/production.log.0.gz | /usr/local/bin/pl_analyze /dev/stdin
+
+Or, have pl_analyze email you (which is preferred, because tabs get preserved):
+
+  /usr/bin/gzip -dc /var/log/production.log.0.gz | /usr/local/bin/pl_analyze /dev/stdin -e devnull@robotcoop.com -s "pl_analyze for `date -v-1d "+%D"`"
+
+In the future, pl_analyze will be able to read from STDIN.
+
+== Sample output
+
+  Request Times Summary:          Count   Avg     Std Dev Min     Max
+  ALL REQUESTS:                   11      0.576   0.508   0.000   1.470
+  
+  ThingsController#view:          3       0.716   0.387   0.396   1.260
+  TeamsController#progress:       2       0.841   0.629   0.212   1.470
+  RssController#uber:             2       0.035   0.000   0.035   0.035
+  PeopleController#progress:      2       0.489   0.489   0.000   0.977
+  PeopleController#view:          2       0.731   0.371   0.360   1.102
+  
+  Average Request Time: 0.634
+  Request Time Std Dev: 0.498
+  
+  Slowest Request Times:
+          TeamsController#progress took 1.470s
+          ThingsController#view took 1.260s
+          PeopleController#view took 1.102s
+          PeopleController#progress took 0.977s
+          ThingsController#view took 0.492s
+          ThingsController#view took 0.396s
+          PeopleController#view took 0.360s
+          TeamsController#progress took 0.212s
+          RssController#uber took 0.035s
+          RssController#uber took 0.035s
+  
+  ------------------------------------------------------------------------
+  
+  DB Times Summary:               Count   Avg     Std Dev Min     Max
+  ALL REQUESTS:                   11      0.366   0.393   0.000   1.144
+  
+  ThingsController#view:          3       0.403   0.362   0.122   0.914
+  TeamsController#progress:       2       0.646   0.497   0.149   1.144
+  RssController#uber:             2       0.008   0.000   0.008   0.008
+  PeopleController#progress:      2       0.415   0.415   0.000   0.830
+  PeopleController#view:          2       0.338   0.149   0.189   0.486
+  
+  Average DB Time: 0.402
+  DB Time Std Dev: 0.394
+  
+  Slowest Total DB Times:
+          TeamsController#progress took 1.144s
+          ThingsController#view took 0.914s
+          PeopleController#progress took 0.830s
+          PeopleController#view took 0.486s
+          PeopleController#view took 0.189s
+          ThingsController#view took 0.173s
+          TeamsController#progress took 0.149s
+          ThingsController#view took 0.122s
+          RssController#uber took 0.008s
+          RssController#uber took 0.008s
+  
+  ------------------------------------------------------------------------
+  
+  Render Times Summary:           Count   Avg     Std Dev Min     Max
+  ALL REQUESTS:                   11      0.219   0.253   0.000   0.695
+  
+  ThingsController#view:          3       0.270   0.171   0.108   0.506
+  TeamsController#progress:       2       0.000   0.000   0.000   0.000
+  RssController#uber:             2       0.012   0.000   0.012   0.012
+  PeopleController#progress:      2       0.302   0.302   0.000   0.604
+  PeopleController#view:          2       0.487   0.209   0.278   0.695
+  
+  Average Render Time: 0.302
+  Render Time Std Dev: 0.251
+  
+  Slowest Total Render Times:
+          PeopleController#view took 0.695s
+          PeopleController#progress took 0.604s
+          ThingsController#view took 0.506s
+          PeopleController#view took 0.278s
+          ThingsController#view took 0.197s
+          ThingsController#view took 0.108s
+          RssController#uber took 0.012s
+          RssController#uber took 0.012s
+          TeamsController#progress took 0.000s
+          TeamsController#progress took 0.000s
+
+== What's missing
+
+* More reports
+* Command line arguments including:
+  * Help
+  * What type of log file you've got (if somebody sends patches with tests)
+* Read from STDIN
+

data/Rakefile

@@ -0,0 +1,17 @@
+require 'hoe'
+
+$:.unshift './lib'
+require 'production_log/analyzer'
+
+Hoe.new 'production_log_analyzer', '2009022403' do |p|
+  p.summary = p.paragraphs_of('README.txt', 1).join ' '
+  p.description = p.paragraphs_of('README.txt', 7).join ' '
+  p.author = 'Eric Hodel'
+  p.email = 'drbrain@segment7.net'
+  p.url = p.paragraphs_of('README.txt', 2).join ' '
+
+  p.rubyforge_name = 'seattlerb'
+
+  p.extra_deps << ['rails_analyzer_tools', '>= 1.4.0']
+end
+

data/bin/action_errors

@@ -0,0 +1,46 @@
+#!/usr/local/bin/ruby -ws
+
+$h ||= false
+$r ||= false
+$o ||= false
+
+$r = $r ? ($r.to_i rescue false) : false
+
+if $h then
+  $stderr.puts "Usage: #{$0} [-r=N] LOGFILE"
+  $stderr.puts "\t-r=N\tShow routing errors with N or more occurances"
+  $stderr.puts "\t-o\tShow errors with one occurance"
+  exit
+end
+
+errors = {}
+counts = Hash.new 0
+
+ARGF.each_line do |line|
+  line =~ /\]: (.*?)     (.*)/
+  next if $1.nil?
+  msg = $1
+  trace = $2
+  key = msg.gsub(/\d/, '#')
+  counts[key] += 1
+  next if counts[key] > 1
+  trace = trace.split('     ')[0..-2].map { |l| l.strip }.join("\n\t")
+  error = "#{msg}\n\t#{trace}"
+  errors[key] = error
+end
+
+counts.sort_by { |_,c| -c }.each do |key, count|
+  next if count == 1 and not $o
+  error = errors[key]
+
+  if error =~ /^ActionController::RoutingError/ then
+    next unless $r
+    next if $r and count < $r
+  end
+
+  puts "count: #{count}"
+  puts "{{{"
+  puts error
+  puts "}}}"
+end
+

data/bin/action_grep

@@ -0,0 +1,19 @@
+#!/usr/local/bin/ruby -w
+
+require 'production_log/action_grep'
+
+action_name = ARGV.shift
+file_name = ARGV.shift
+
+if action_name.nil? or file_name.nil? then
+  puts "Usage: #{$0} action_name file_name"
+  exit 1
+end
+
+begin
+  ActionGrep.grep action_name, file_name
+rescue ArgumentError => e
+  puts e
+  exit 1
+end
+

data/bin/pl_analyze

@@ -0,0 +1,36 @@
+#!/bin/env ruby -w
+
+$:.unshift "#{File.dirname(__FILE__)}/../lib/"
+require "production_log/analyzer"
+
+file_name = ARGV.shift
+
+if file_name.nil? then
+  puts "Usage: #{$0} file_name [-e email_recipient [-s subject]] [count]"
+  exit 1
+end
+
+email_recipient = nil
+subject = nil
+
+if ARGV.first == '-e' then
+  ARGV.shift # -e
+  email_recipient = ARGV.shift
+end
+
+if email_recipient and ARGV.first == '-s' then
+  ARGV.shift # -s
+  subject = ARGV.shift
+end
+
+count = ARGV.shift
+count = count.nil? ? 10 : Integer(count)
+
+if email_recipient.nil? then
+  analyzer = Analyzer.new file_name
+  analyzer.process
+  puts analyzer.report(count)
+else
+  Analyzer.email file_name, email_recipient, subject, count
+end
+

data/lib/passenger_log_per_proc.rb

@@ -0,0 +1,55 @@
+class PassengerLogPerProc
+  
+  cattr_accessor :log_path_prefix
+  
+  def self.logger_mutex
+   @@logger_mutex ||= Mutex.new
+  end
+  
+  # with log_path_with_prefix like {RAILS_ROOT}/log/passenger/{RAILS_ENV}
+  # you will get logs like {RAILS_ROOT}/log/passenger/{RAILS_ENV}_{Pid}.log
+  def self.enable(log_path_with_prefix)
+    PassengerLogPerProc.log_path_prefix = log_path_with_prefix
+    PhusionPassenger::Rack::RequestHandler.class_eval do
+      cattr_accessor :process_logger
+      
+      def process_request_with_extra_logging(env, input, output)
+        unless self.class.process_logger
+          PhusionPassenger::Rack::RequestHandler.process_logger = ActiveSupport::BufferedLogger.new(PassengerLogPerProc.log_path_prefix + "_#{Process.pid}" + ".log")
+
+          PhusionPassenger::Rack::RequestHandler.process_logger.debug("\nPassenger logging started")
+
+          RAILS_DEFAULT_LOGGER.instance_eval do
+            class << self
+              def add(severity, message = nil, progname = nil, &block)
+                to_return = super
+                if to_return
+                  Thread.current[:passenger_logs] ||= []
+                  Thread.current[:passenger_logs] << [severity, to_return.strip]
+                end
+                to_return
+              end
+            end
+          end          
+        end
+        
+        Thread.current[:passenger_logs] ||= []
+        start_time = Time.now
+        process_request_without_extra_logging(env, input, output)
+      ensure
+        PassengerLogPerProc.logger_mutex.synchronize do
+          PhusionPassenger::Rack::RequestHandler.process_logger.info("RECEIVE_REQUEST (#{Process.pid}) #{env['REQUEST_METHOD']} #{env['REQUEST_URI']} #{start_time}")        
+          Thread.current[:passenger_logs].each do |to_log|
+            PhusionPassenger::Rack::RequestHandler.process_logger.add(to_log[0], to_log[1])
+          end
+          PhusionPassenger::Rack::RequestHandler.process_logger.info("#{Time.now} (#{Process.pid}) RESPONSE_SENT")
+          PhusionPassenger::Rack::RequestHandler.process_logger.info("\n")
+          Thread.current[:passenger_logs] = []
+        end
+      end
+  
+      alias_method_chain :process_request, :extra_logging  
+    end    
+  end
+    
+end

data/lib/production_log/action_grep.rb

@@ -0,0 +1,41 @@
+module ActionGrep; end
+
+class << ActionGrep
+
+  def grep(action_name, file_name)
+    unless action_name =~ /\A([A-Z][A-Za-z\d]*)(?:#([A-Za-z]\w*))?\Z/ then
+      raise ArgumentError, "Invalid action name #{action_name} expected something like SomeController#action"
+    end
+
+    unless File.file? file_name and File.readable? file_name then
+      raise ArgumentError, "Unable to read #{file_name}"
+    end
+
+    buckets = Hash.new { |h,k| h[k] = [] }
+    comp_count = Hash.new 0
+
+    File.open file_name do |fp|
+      LogParser.detect_mode(fp)
+      fp.each_line do |line|
+        bucket, data = LogParser.extract_bucket_and_data(line)
+        next if !bucket
+
+        buckets[bucket] << line
+
+        case data
+        when /^Start rendering component / then
+          comp_count[bucket] += 1
+        when /^End of component rendering$/ then
+          comp_count[bucket] -= 1
+        when /^Completed/ then
+          next unless comp_count[bucket] == 0
+          action = buckets.delete bucket
+          next unless action.any? { |l| l =~ /Processing #{action_name}/ }
+          puts action.join
+        end
+      end
+    end
+  end
+
+end
+