production_log_analyzer 1.2.1 → 1.3.0

data/Manifest.txt

@@ -2,10 +2,13 @@ Rakefile
 README
 LICENSE
 Manifest.txt
+bin/action_grep
 bin/pl_analyze
+lib/production_log/action_grep.rb
 lib/production_log/analyzer.rb
 lib/production_log/parser.rb
-lib/production_log/syslog_logger.rb
+test/test.syslog.0.14.x.log
 test/test.syslog.log
+test/test_action_grep.rb
 test/test_analyzer.rb
 test/test_parser.rb

data/README

@@ -6,19 +6,21 @@ http://rubyforge.org/projects/rails-analyzer
 
 = About
 
-The Production Analyzer lets you find out which pages on your site are
-dragging you down.  PL Analyze requires the use of SyslogLogger (included)
-because the default Logger doesn't give any way to associate lines logged to a
-request.
+The Production Log Analyzer lets you find out which pages on your site are
+dragging you down.  The PL Analyzer requires the use of SyslogLogger from
+rails_analyzer_tools because the default Logger doesn't give any way to
+associate lines logged to a request.
 
-SyslogLogger gives you many other advantages, such as the ability to combine
-logs from multiple machines.
+The PL Analyzer also includes action_grep which lets you grab lines from a log
+that only match a single action.
+
+  action_grep RssController#uber /var/log/production.log
 
 = Installing
 
 == Download
 
-Grab the gem (or tar.gz or .zip) from the RubyForge project:
+Grab the gem (or tar.gz) from the RubyForge project:
 
 http://rubyforge.org/frs/?group_id=586
 
@@ -27,8 +29,8 @@ http://rubyforge.org/frs/?group_id=586
 Either:
 
 A syslogd that doesn't suck.  This means that syslog.conf(5) shows a
-!prog specification.  (FreeBSD's syslogd doesn't suck, but OS X's syslogd
-does.)
+!program_name specification.  (FreeBSD's syslogd doesn't suck, but OS X's
+syslogd does.)
 
 or:
 
@@ -40,13 +42,16 @@ submit patches with tests.  (Patches without tests are useless to me.)
 
 Either:
 
-Use SyslogLogger according to directions on the SyslogLogger page, including
-setting up your non-sucky syslogd as directed.
+Use SyslogLogger according to directions on the SyslogLogger RDoc page,
+including setting up your non-sucky syslogd as directed.  (SyslogLogger is
+included via the rails_analyzer_tools gem.)
+
+http://rails-analyzer.rubyforge.org/tools/classes/SyslogLogger.html
 
 or:
 
 Use your hacking skills and set up a logger that your hacked LogParser#parse
-can deal with.
+can deal with.  (Then send my your patches for integration.)
 
 Then:
 

data/Rakefile

@@ -8,55 +8,56 @@ require 'rake/contrib/sshpublisher'
 $VERBOSE = nil
 
 spec = Gem::Specification.new do |s|
-    s.name = "production_log_analyzer"
-    s.version = "1.2.1"
-    s.summary = "Extracts statistics from Rails production logs"
-    s.author = "Eric Hodel"
-    s.email = "hodel@robotcoop.com"
-
-    s.has_rdoc = true
-    s.files = File.read("Manifest.txt").split($/)
-    s.require_path = 'lib'
-    s.executables = ["pl_analyze"]
-    s.default_executable = "pl_analyze"
+  s.name = 'production_log_analyzer'
+  s.version = '1.3.0'
+  s.summary = 'Extracts statistics from Rails production logs'
+  s.author = 'Eric Hodel'
+  s.email = 'hodel@robotcoop.com'
+
+  s.has_rdoc = true
+  s.files = File.read('Manifest.txt').split($/)
+  s.require_path = 'lib'
+  s.executables = ['pl_analyze', 'action_grep']
+  s.default_executable = 'pl_analyze'
+
+  s.add_dependency 'rails_analyzer_tools', '>= 1.1.0'
 end
 
-desc "Run tests"
+desc 'Run tests'
 task :default => [ :test ]
 
-Rake::TestTask.new("test") do |t|
-    t.libs << "test"
-    t.pattern = "test/test_*.rb"
-    t.verbose = true
+Rake::TestTask.new('test') do |t|
+  t.libs << 'test'
+  t.pattern = 'test/test_*.rb'
+  t.verbose = true
 end
 
-desc "Generate RDoc"
+desc 'Generate RDoc'
 Rake::RDocTask.new :rdoc do |rd|
-    rd.rdoc_dir = "doc"
-    rd.rdoc_files.add "lib", "README", "LICENSE"
-    rd.main = "README"
-    rd.options << "-d" if `which dot` =~ /\/dot/
+  rd.rdoc_dir = 'doc'
+  rd.rdoc_files.add 'lib', 'README', 'LICENSE'
+  rd.main = 'README'
+  rd.options << '-d' if `which dot` =~ /\/dot/
 end
 
-desc "Build Gem"
+desc 'Build Gem'
 Rake::GemPackageTask.new spec do |pkg|
-    pkg.need_zip = true
-    pkg.need_tar = true
+  pkg.need_tar = true
 end
 
-desc "Sends RDoc to RubyForge"
+desc 'Sends RDoc to RubyForge'
 task :send_rdoc => [ :rerdoc ] do
-    publisher = Rake::SshDirPublisher.new('drbrain@rubyforge.org',
-                    '/var/www/gforge-projects/rails-analyzer',
-                    'doc')
-    publisher.upload
+  publisher = Rake::SshDirPublisher.new('drbrain@rubyforge.org',
+                                        '/var/www/gforge-projects/rails-analyzer/pl_analyze',
+                                        'doc')
+  publisher.upload
 end
 
-desc "Clean up"
+desc 'Clean up'
 task :clean => [ :clobber_rdoc, :clobber_package ]
 
-desc "Clean up"
+desc 'Clean up'
 task :clobber => [ :clean ]
 
-# vim: ts=4 sts=4 sw=4 syntax=Ruby
+# vim: syntax=Ruby
 

data/bin/action_grep

@@ -0,0 +1,19 @@
+#!/usr/local/bin/ruby -w
+
+require 'production_log/action_grep'
+
+action_name = ARGV.shift
+file_name = ARGV.shift
+
+if action_name.nil? or file_name.nil? then
+  puts "Usage: #{$0} action_name file_name"
+  exit 1
+end
+
+begin
+  ActionGrep.grep action_name, file_name
+rescue ArgumentError => e
+  puts e
+  exit 1
+end
+

data/lib/production_log/action_grep.rb

@@ -0,0 +1,39 @@
+module ActionGrep; end
+
+class << ActionGrep
+
+  def grep(action_name, file_name)
+    unless action_name =~ /\A([A-Z][A-Za-z\d]*)(?:#([A-Za-z]\w*))?\Z/ then
+      raise ArgumentError, "Invalid action name #{action_name} expected something like FooController#uber"
+    end
+
+    unless File.file? file_name and File.readable? file_name then
+      raise ArgumentError, "Unable to read #{file_name}"
+    end
+
+    in_component = 0
+    buckets = Hash.new { |h,k| h[k] = [] }
+    File.open file_name do |fp|
+      fp.each_line do |line|
+        line =~ / ([^ ]+) ([^ ]+)\[(\d+)\]: /
+        bucket = [$1, $2, $3].join '-'
+
+        buckets[bucket] << line
+
+        case line
+        when /\]: Start rendering component / then
+          in_component += 1
+        when /\]: End of component rendering$/ then
+          in_component -= 1
+        when /\]: Completed/ then
+          next unless in_component == 0
+          action = buckets.delete bucket
+          next unless action.first =~ /#{action_name}/
+          puts action.join
+        end
+      end
+    end
+  end
+
+end
+

data/lib/production_log/parser.rb

@@ -61,6 +61,7 @@ module LogParser
             @request_time = 0
             @render_time = 0
             @db_time = 0
+            @in_component = 0
 
             parse entry
         end
@@ -75,20 +76,28 @@ module LogParser
                 when /^Parameters/, /^Cookie set/, /^Rendering/,
                      /^Redirected/ then
                     # nothing
-                when /^Processing ([\S]+) \(for (.+) at (.*)\)$/ then
+                when /^Processing ([\S]+) \(for (.+) at (.*)\)/ then
+                    next if @in_component > 0
                     @page = $1
                     @ip   = $2
                     @time = $3
                 when /^Completed in ([\S]+) .+ Rendering: ([\S]+) .+ DB: ([\S]+)/ then
+                    next if @in_component > 0
                     @request_time = $1.to_f
                     @render_time = $2.to_f
                     @db_time = $3.to_f
                 when /^Completed in ([\S]+) .+ DB: ([\S]+)/ then # Redirect
+                    next if @in_component > 0
                     @request_time = $1.to_f
                     @render_time = 0
                     @db_time = $2.to_f
                 when /(.+?) \(([^)]+)\)   / then
                     @queries << [$1, $2.to_f]
+                when /^Start rendering component / then
+                    @in_component += 1
+                when /^End of component rendering$/ then
+                    @in_component -= 1
+                when /^Fragment hit: / then
                 else
                     raise "Can't handle #{line.inspect}" if $TESTING
                 end
@@ -116,16 +125,25 @@ module LogParser
     # the same process id through Completed.
 
     def self.parse(stream) # :yields: log_entry
+        in_component = 0
         buckets = Hash.new { |h,k| h[k] = [] }
+        comp_count = Hash.new 0
+
         stream.each_line do |line|
             line =~ / ([^ ]+) ([^ ]+)\[(\d+)\]: (.*)/
-            next if $2 != 'rails'
+            next if $2.nil? or $2 == 'newsyslog'
             bucket = "#{$1}-#{$3}"
             data = $4
 
             buckets[bucket] << data
 
-            if data =~ /^Completed/ then
+            case data
+            when /^Start rendering component / then
+                comp_count[bucket] += 1
+            when /^End of component rendering$/ then
+                comp_count[bucket] -= 1
+            when /^Completed/ then
+                next unless comp_count[bucket] == 0
                 entry = buckets.delete bucket
                 if entry.first =~ /^Processing/ then
                     yield LogEntry.new(entry)

data/test/test.syslog.0.14.x.log

@@ -0,0 +1,4 @@
+Nov  7 12:14:02 192.168.1.71 43things[24950]: Processing RssController#entries (for 10.43.199.12 at 2005-11-07 12:14:02) [GET]
+Nov  7 12:14:02 192.168.1.71 43things[24950]: Parameters: {"rss/entries/goal.html/entries/goal"=>nil, "action"=>"entries", "id"=>"goal", "controller"=>"rss", "goal_id"=>"86381"}
+Nov  7 12:14:02 192.168.1.71 43things[24950]: Rendering rss/rss2.0
+Nov  7 12:14:02 192.168.1.71 43things[24950]: Completed in 0.00860 (116 reqs/sec) | DB: 0.00328 (38%) | 200 OK [http://www.43things.com/rss/entries/goal?goal_id=86381]

data/test/test_action_grep.rb

@@ -0,0 +1,68 @@
+#!/usr/local/bin/ruby -w
+
+require 'stringio'
+require 'tempfile'
+require 'test/unit'
+
+require 'production_log/action_grep'
+
+class TestActionGrep < Test::Unit::TestCase
+
+  def setup
+    @syslog_file_name = File.expand_path(File.join(File.dirname(__FILE__),
+                                                   'test.syslog.log'))
+  end
+
+  def test_module_grep
+    old_stdout = $stdout.dup
+    stdout = StringIO.new
+    $stdout = stdout
+
+    ActionGrep.grep 'RssController', @syslog_file_name
+
+    stdout.rewind
+
+    lines = stdout.readlines
+
+    assert_equal 18, lines.length
+
+  ensure
+    $stdout = old_stdout
+  end
+
+  def test_module_grep_arguments
+    file = Tempfile.new File.basename(__FILE__)
+
+    assert_raises ArgumentError do
+      ActionGrep.grep 'Foo_Controller', '/tmp/no_such_file/no_really/'
+    end
+
+    assert_raises ArgumentError do
+      ActionGrep.grep 'FooController#5', '/tmp/no_such_file/no_really/'
+    end
+
+    assert_raises ArgumentError do
+      ActionGrep.grep '5', '/tmp/no_such_file/no_really/'
+    end
+
+    assert_raises ArgumentError do
+      ActionGrep.grep 'FooController', '/tmp/no_such_file/no_really'
+    end
+
+    assert_nothing_raised do
+      ActionGrep.grep 'FooController', file.path
+      ActionGrep.grep 'FooController5', file.path
+      ActionGrep.grep 'FooController#action', file.path
+      ActionGrep.grep 'FooController#action_thingy', file.path
+      ActionGrep.grep 'FooController#action_thingy5', file.path
+      ActionGrep.grep 'FooController5#action', file.path
+      ActionGrep.grep 'FooController5#action_thingy', file.path
+      ActionGrep.grep 'FooController5#action_thingy5', file.path
+    end
+
+  ensure
+    file.close
+  end
+
+end
+

data/test/test_parser.rb

@@ -92,7 +92,7 @@ end
 class TestLogParser <Test::Unit::TestCase
 
     def test_class_parse
-        log = StringIO.new <<EOF
+        log = StringIO.new <<-EOF
 Mar  7 00:00:25 online1 rails[59628]: Processing RssController#uber (for 67.18.200.5 at Mon Mar 07 00:00:25 CST 2005)
 Mar  7 00:00:25 online1 rails[59628]: Parameters: {:id=>"author", :"rss/uber/author.html/uber/author"=>nil, :action=>"uber", :username=>"looch", :controller=>"rss"}
 Mar  7 00:00:25 online1 rails[59628]: Cookie set: auth=dc%2FGUP20BwziF%2BApGecc0pXB0PF0obi55az63ubAFtsnOOdJPkhfJH2U09yuzQD3WtdmWnydLzFcRA78kwi7Gw%3D%3D; path=/; expires=Thu, 05 Mar 2015 06:00:25 GMT
@@ -102,7 +102,7 @@ Mar  7 00:00:25 online1 rails[59628]: Person Load (0.002445)   SELECT * FROM peo
 Mar  7 00:00:25 online1 rails[59628]: ProfileImage Load (0.001554)   SELECT * FROM profile_images WHERE id = 2782 LIMIT 1
 Mar  7 00:00:25 online1 rails[59628]: Rendering rss/rss2.0 (200 OK)
 Mar  7 00:00:25 online1 rails[59628]: Completed in 0.034519 (28 reqs/sec) | Rendering: 0.011770 (34%) | DB: 0.007962 (23%)
-EOF
+        EOF
 
         entries = []
 
@@ -114,6 +114,36 @@ EOF
         assert_equal 'RssController#uber', entries.first.page
     end
 
+    def test_class_parse_components
+        log = StringIO.new <<-EOF
+Jul 11 10:05:20 www rails[61243]: Processing ChatroomsController#launch (for 213.152.37.169 at Mon Jul 11 10:05:20 CDT 2005)
+Jul 11 10:05:20 www rails[61243]: Start rendering component ({:action=>"online_count", :controller=>"members"}):
+Jul 11 10:05:20 www rails[34216]: Processing ChatroomsController#launch (for 213.152.37.169 at Mon Jul 11 10:05:20 CDT 2005)
+Jul 11 10:05:20 www rails[34216]: Start rendering component ({:action=>"online_count", :controller=>"members"}):
+Jul 11 10:05:20 www rails[34216]: Processing MembersController#online_count (for 213.152.37.169 at Mon Jul 11 10:05:20 CDT 2005)
+Jul 11 10:05:20 www rails[34216]: Completed in 0.00741 (135 reqs/sec) | DB: 0.00320 (43%)
+Jul 11 10:05:20 www rails[34216]: End of component rendering
+Jul 11 10:05:28 www rails[34216]: Completed in 8.65005 (0 reqs/sec) | Rendering: 8.64820 (99%) | DB: 0.00000 (0%)
+Jul 11 10:05:20 www rails[34216]: Processing ChatroomsController#launch (for 213.152.37.169 at Mon Jul 11 10:05:20 CDT 2005)
+Jul 11 10:05:20 www rails[34216]: Start rendering component ({:action=>"online_count", :controller=>"members"}):
+Jul 11 10:05:20 www rails[34216]: Processing MembersController#online_count (for 213.152.37.169 at Mon Jul 11 10:05:20 CDT 2005)
+Jul 11 10:05:20 www rails[34216]: Completed in 0.00741 (135 reqs/sec) | DB: 0.00320 (43%)
+Jul 11 10:05:20 www rails[34216]: End of component rendering
+Jul 11 10:05:28 www rails[34216]: Completed in 8.65005 (0 reqs/sec) | Rendering: 8.64820 (99%) | DB: 0.00000 (0%)
+Jul 11 10:05:20 www rails[61243]: Processing MembersController#online_count (for 213.152.37.169 at Mon Jul 11 10:05:20 CDT 2005)
+Jul 11 10:05:20 www rails[61243]: Completed in 0.00741 (135 reqs/sec) | DB: 0.00320 (43%)
+Jul 11 10:05:20 www rails[61243]: End of component rendering
+Jul 11 10:05:28 www rails[61243]: Completed in 8.65005 (0 reqs/sec) | Rendering: 8.64820 (99%) | DB: 0.00000 (0%)
+        EOF
+
+        entries = []
+        LogParser.parse(log) { |entry| entries << entry }
+
+        assert_equal 3, entries.length
+        assert_equal 'ChatroomsController#launch', entries.first.page
+        assert_equal 8.65005, entries.first.request_time
+    end
+
     def test_class_parse_multi
         entries = []
         File.open 'test/test.syslog.log' do |fp|
@@ -134,6 +164,15 @@ EOF
         assert_equal 0, last.request_time
     end
 
+    def test_class_parse_0_14_x
+        entries = []
+        File.open 'test/test.syslog.0.14.x.log' do |fp|
+            LogParser.parse fp do |entry|
+                entries << entry
+            end
+        end
+    end
+
 end
 
 # vim: ts=4 sts=4 sw=4

metadata

@@ -1,10 +1,10 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.8.10
+rubygems_version: 0.8.11
 specification_version: 1
 name: production_log_analyzer
 version: !ruby/object:Gem::Version 
-  version: 1.2.1
-date: 2005-05-16
+  version: 1.3.0
+date: 2005-11-08 00:00:00 -08:00
 summary: Extracts statistics from Rails production logs
 require_paths: 
   - lib
@@ -24,6 +24,8 @@ required_ruby_version: !ruby/object:Gem::Version::Requirement
         version: 0.0.0
   version: 
 platform: ruby
+signing_key: 
+cert_chain: 
 authors: 
   - Eric Hodel
 files: 
@@ -31,11 +33,14 @@ files:
   - README
   - LICENSE
   - Manifest.txt
+  - bin/action_grep
   - bin/pl_analyze
+  - lib/production_log/action_grep.rb
   - lib/production_log/analyzer.rb
   - lib/production_log/parser.rb
-  - lib/production_log/syslog_logger.rb
+  - test/test.syslog.0.14.x.log
   - test/test.syslog.log
+  - test/test_action_grep.rb
   - test/test_analyzer.rb
   - test/test_parser.rb
 test_files: []
@@ -43,6 +48,17 @@ rdoc_options: []
 extra_rdoc_files: []
 executables: 
   - pl_analyze
+  - action_grep
 extensions: []
 requirements: []
-dependencies: []
+dependencies: 
+  - !ruby/object:Gem::Dependency 
+    name: rails_analyzer_tools
+    version_requirement: 
+    version_requirements: !ruby/object:Gem::Version::Requirement 
+      requirements: 
+        - 
+          - ">="
+          - !ruby/object:Gem::Version 
+            version: 1.1.0
+      version: 

data/lib/production_log/syslog_logger.rb

@@ -1,150 +0,0 @@
-require 'syslog'
-require 'logger'
-
-##
-# This Logger logs messages to syslogd.  Add it to your production environment
-# so you can run pl_analyze on your log files.
-#
-# SyslogLogger logs everything.  Tweak your syslog.conf to throw away messages
-# you don't want, SyslogLogger ignores any calls to #level=.
-# 
-# SyslogLogger logs messages using the 'rails' facility.
-#
-# = Sample usage
-# 
-# == config/environment/production.rb
-#
-# Add the following lines:
-# 
-#   require 'production_log/syslog_logger'
-#   RAILS_DEFAULT_LOGGER = SyslogLogger.new
-#
-# == config/environment.rb
-#
-# In 0.10.0, change this line:
-# 
-#   RAILS_DEFAULT_LOGGER = Logger.new("#{RAILS_ROOT}/log/#{RAILS_ENV}.log")
-#
-# to:
-#
-#   RAILS_DEFAULT_LOGGER ||= Logger.new("#{RAILS_ROOT}/log/#{RAILS_ENV}.log")
-#
-# Other versions of Rails should have a similar change.
-#
-# == /etc/syslog.conf
-#
-# Add the following lines:
-# 
-#  !rails
-#  *.*                                             /var/log/production.log
-#
-# Then touch /var/log/production.log and signal syslogd with a HUP
-# (killall -HUP syslogd, on FreeBSD).
-#
-# == /etc/newsyslog.conf
-#
-# Add the following line:
-# 
-#   /var/log/production.log                 640  7     *    @T00  Z
-# 
-# This creates a log file that is rotated every day at midnight, gzip'd, then
-# kept for 7 days.  Consult newsyslog.conf(5) for more details.
-#
-# Now restart your Rails app.  Your production logs should now be showing up
-# in /var/log/production.log.  If you have mulitple machines, you can log them
-# all to a central machine with remote syslog logging for analysis.
-
-class SyslogLogger
-
-    ##
-    # Log level for Logger compatibility.
-
-    attr_accessor :level
-
-    ##
-    # Syslog is a module, so this constant holds our open logger.
-
-    SYSLOG = Syslog.open "rails"
-
-    ##
-    # Maps Logger warning types to syslog(3) warning types.
-
-    LOGGER_MAP = {
-        :fatal => :err,
-        :error => :warning,
-        :warn  => :notice,
-        :info  => :info,
-        :debug => :debug,
-    }
-
-    ##
-    # Maps Logger log levels to their values so we can silence.
-
-    LOGGER_LEVEL_MAP = {}
-
-    LOGGER_MAP.each_key do |key|
-        LOGGER_LEVEL_MAP[key] = Logger.const_get key.to_s.upcase
-    end
-
-    ##
-    # Builds a logging method for level +meth+.
-
-    def self.log_method(meth)
-        eval <<-EOF
-            def #{meth}(message)
-                SYSLOG.#{LOGGER_MAP[meth]} clean(message) if #{LOGGER_LEVEL_MAP[meth]} >= @level
-            end
-        EOF
-    end
-
-    LOGGER_MAP.each_key do |level|
-        log_method level
-    end
-
-    ##
-    # Fill in variables for Logger compatibility.
-
-    def initialize(*args)
-        @level = Logger::DEBUG
-    end
-
-    ##
-    # Allows messages of a particular log level to be ignored temporarily.
-    #
-    # Can you say "Broken Windows"?
-
-    def silence(temporary_level = Logger::ERROR)
-        old_logger_level = @level
-        @level = temporary_level 
-        yield
-    ensure
-        @level = old_logger_level
-    end
-
-    private
-
-    ##
-    # Clean up messages so they're nice and pretty.
-
-    def clean(message)
-        message = message.dup
-        message.strip!
-        message.gsub!(/%/, '%%') # syslog(3) freaks on % (printf)
-        message.gsub!(/\e\[[^m]*m/, '') # remove useless ansi color codes
-        return message
-    end
-
-end
-
-if __FILE__ == $0 then
-    logger = SyslogLogger.new
-
-    message = "message from syslog_logger.rb logged using"
-
-    SyslogLogger::LOGGER_MAP.each do |logger_level, syslog_level|
-        logger.send logger_level, "#{logger_level} #{message} #{syslog_level}"
-    end
-end
-
-# vim: ts=4 sts=4 sw=4
-