brontes3d-production_log_analyzer 2009022403
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +34 -0
- data/LICENSE.txt +27 -0
- data/Manifest.txt +18 -0
- data/README.txt +147 -0
- data/Rakefile +17 -0
- data/bin/action_errors +46 -0
- data/bin/action_grep +19 -0
- data/bin/pl_analyze +36 -0
- data/lib/passenger_log_per_proc.rb +55 -0
- data/lib/production_log/action_grep.rb +41 -0
- data/lib/production_log/analyzer.rb +416 -0
- data/lib/production_log/parser.rb +228 -0
- data/test/test_action_grep.rb +72 -0
- data/test/test_analyzer.rb +425 -0
- data/test/test_helper.rb +68 -0
- data/test/test_parser.rb +420 -0
- data/test/test_passenger_log_per_proc.rb +88 -0
- data/test/test_syslogs/test.syslog.0.14.x.log +4 -0
- data/test/test_syslogs/test.syslog.1.2.shortname.log +4 -0
- data/test/test_syslogs/test.syslog.empty.log +0 -0
- data/test/test_syslogs/test.syslog.log +256 -0
- data/test/test_vanilla/test.0.14.x.log +4 -0
- data/test/test_vanilla/test.1.2.shortname.log +4 -0
- data/test/test_vanilla/test.empty.log +0 -0
- data/test/test_vanilla/test.log +255 -0
- data/test/test_vanilla/test_log_parts/1_online1-rails-59600.log +7 -0
- data/test/test_vanilla/test_log_parts/2_online2-rails-59628.log +11 -0
- data/test/test_vanilla/test_log_parts/3_online1-rails-59628.log +9 -0
- data/test/test_vanilla/test_log_parts/4_online1-rails-59645.log +30 -0
- data/test/test_vanilla/test_log_parts/5_online1-rails-59629.log +38 -0
- data/test/test_vanilla/test_log_parts/6_online1-rails-60654.log +32 -0
- data/test/test_vanilla/test_log_parts/7_online1-rails-59627.log +70 -0
- data/test/test_vanilla/test_log_parts/8_online1-rails-59635.log +58 -0
- metadata +113 -0
data/History.txt
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
= 1.5.0
|
2
|
+
|
3
|
+
* Fixed empty log bug. Patch by Tim Lucas.
|
4
|
+
* Fixed bug where sometimes lines would be logged before the
|
5
|
+
Processing line. Patch by Geoff Grosenbach.
|
6
|
+
|
7
|
+
= 1.4.0
|
8
|
+
|
9
|
+
* Switched to Hoe
|
10
|
+
* Allowed action_errors to suppress routing errors with > 3 occurances
|
11
|
+
* action_grep now works correctly with components
|
12
|
+
* pl_analyze now works correctly with components
|
13
|
+
* Added action_errors to extract error counts from logs
|
14
|
+
* Retabbed to match the rest of the world
|
15
|
+
|
16
|
+
= 1.3.0
|
17
|
+
|
18
|
+
* Added action_grep
|
19
|
+
* Added support for newer log format
|
20
|
+
|
21
|
+
= 1.2.0
|
22
|
+
|
23
|
+
* pl_analyze calculates per-action statistics
|
24
|
+
* pl_analyze can send an email with its output
|
25
|
+
|
26
|
+
= 1.1.0
|
27
|
+
|
28
|
+
* RDoc
|
29
|
+
* Other various fixes lost to time.
|
30
|
+
|
31
|
+
= 1.0.0
|
32
|
+
|
33
|
+
* Birthday!
|
34
|
+
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
Copyright 2005, 2007 Eric Hodel, The Robot Co-op. All rights reserved.
|
2
|
+
|
3
|
+
Redistribution and use in source and binary forms, with or without
|
4
|
+
modification, are permitted provided that the following conditions
|
5
|
+
are met:
|
6
|
+
|
7
|
+
1. Redistributions of source code must retain the above copyright
|
8
|
+
notice, this list of conditions and the following disclaimer.
|
9
|
+
2. Redistributions in binary form must reproduce the above copyright
|
10
|
+
notice, this list of conditions and the following disclaimer in the
|
11
|
+
documentation and/or other materials provided with the distribution.
|
12
|
+
3. Neither the names of the authors nor the names of their contributors
|
13
|
+
may be used to endorse or promote products derived from this software
|
14
|
+
without specific prior written permission.
|
15
|
+
|
16
|
+
THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS
|
17
|
+
OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
18
|
+
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
19
|
+
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
|
20
|
+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
|
21
|
+
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
|
22
|
+
OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
|
23
|
+
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
24
|
+
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
25
|
+
OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
26
|
+
EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
27
|
+
|
data/Manifest.txt
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
History.txt
|
2
|
+
LICENSE.txt
|
3
|
+
Manifest.txt
|
4
|
+
README.txt
|
5
|
+
Rakefile
|
6
|
+
bin/action_errors
|
7
|
+
bin/action_grep
|
8
|
+
bin/pl_analyze
|
9
|
+
lib/production_log/action_grep.rb
|
10
|
+
lib/production_log/analyzer.rb
|
11
|
+
lib/production_log/parser.rb
|
12
|
+
test/test.syslog.0.14.x.log
|
13
|
+
test/test.syslog.1.2.shortname.log
|
14
|
+
test/test.syslog.empty.log
|
15
|
+
test/test.syslog.log
|
16
|
+
test/test_action_grep.rb
|
17
|
+
test/test_analyzer.rb
|
18
|
+
test/test_parser.rb
|
data/README.txt
ADDED
@@ -0,0 +1,147 @@
|
|
1
|
+
= production_log_analyzer
|
2
|
+
|
3
|
+
production_log_analyzer lets you find out which actions on a Rails
|
4
|
+
site are slowing you down.
|
5
|
+
|
6
|
+
http://seattlerb.rubyforge.org/production_log_analyzer
|
7
|
+
|
8
|
+
http://rubyforge.org/projects/seattlerb
|
9
|
+
|
10
|
+
Bug reports:
|
11
|
+
|
12
|
+
http://rubyforge.org/tracker/?func=add&group_id=1513&atid=5921
|
13
|
+
|
14
|
+
== About
|
15
|
+
|
16
|
+
production_log_analyzer provides three tools to analyze log files
|
17
|
+
created by SyslogLogger. pl_analyze for getting daily reports,
|
18
|
+
action_grep for pulling log lines for a single action and
|
19
|
+
action_errors to summarize errors with counts.
|
20
|
+
|
21
|
+
The analyzer currently requires the use of SyslogLogger because the
|
22
|
+
default Logger doesn't give any way to associate lines logged to a
|
23
|
+
request.
|
24
|
+
|
25
|
+
The PL Analyzer also includes action_grep which lets you grab lines from a log
|
26
|
+
that only match a single action.
|
27
|
+
|
28
|
+
action_grep RssController#uber /var/log/production.log
|
29
|
+
|
30
|
+
== Installing
|
31
|
+
|
32
|
+
sudo gem install production_log_analyzer
|
33
|
+
|
34
|
+
=== Setup
|
35
|
+
|
36
|
+
First:
|
37
|
+
|
38
|
+
Set up SyslogLogger according to the instructions here:
|
39
|
+
|
40
|
+
http://seattlerb.rubyforge.org/SyslogLogger/
|
41
|
+
|
42
|
+
Then:
|
43
|
+
|
44
|
+
Set up a cronjob (or something like that) to run log files through pl_analyze.
|
45
|
+
|
46
|
+
== Using pl_analyze
|
47
|
+
|
48
|
+
To run pl_analyze simply give it the name of a log file to analyze.
|
49
|
+
|
50
|
+
pl_analyze /var/log/production.log
|
51
|
+
|
52
|
+
If you want, you can run it from a cron something like this:
|
53
|
+
|
54
|
+
/usr/bin/gzip -dc /var/log/production.log.0.gz | /usr/local/bin/pl_analyze /dev/stdin
|
55
|
+
|
56
|
+
Or, have pl_analyze email you (which is preferred, because tabs get preserved):
|
57
|
+
|
58
|
+
/usr/bin/gzip -dc /var/log/production.log.0.gz | /usr/local/bin/pl_analyze /dev/stdin -e devnull@robotcoop.com -s "pl_analyze for `date -v-1d "+%D"`"
|
59
|
+
|
60
|
+
In the future, pl_analyze will be able to read from STDIN.
|
61
|
+
|
62
|
+
== Sample output
|
63
|
+
|
64
|
+
Request Times Summary: Count Avg Std Dev Min Max
|
65
|
+
ALL REQUESTS: 11 0.576 0.508 0.000 1.470
|
66
|
+
|
67
|
+
ThingsController#view: 3 0.716 0.387 0.396 1.260
|
68
|
+
TeamsController#progress: 2 0.841 0.629 0.212 1.470
|
69
|
+
RssController#uber: 2 0.035 0.000 0.035 0.035
|
70
|
+
PeopleController#progress: 2 0.489 0.489 0.000 0.977
|
71
|
+
PeopleController#view: 2 0.731 0.371 0.360 1.102
|
72
|
+
|
73
|
+
Average Request Time: 0.634
|
74
|
+
Request Time Std Dev: 0.498
|
75
|
+
|
76
|
+
Slowest Request Times:
|
77
|
+
TeamsController#progress took 1.470s
|
78
|
+
ThingsController#view took 1.260s
|
79
|
+
PeopleController#view took 1.102s
|
80
|
+
PeopleController#progress took 0.977s
|
81
|
+
ThingsController#view took 0.492s
|
82
|
+
ThingsController#view took 0.396s
|
83
|
+
PeopleController#view took 0.360s
|
84
|
+
TeamsController#progress took 0.212s
|
85
|
+
RssController#uber took 0.035s
|
86
|
+
RssController#uber took 0.035s
|
87
|
+
|
88
|
+
------------------------------------------------------------------------
|
89
|
+
|
90
|
+
DB Times Summary: Count Avg Std Dev Min Max
|
91
|
+
ALL REQUESTS: 11 0.366 0.393 0.000 1.144
|
92
|
+
|
93
|
+
ThingsController#view: 3 0.403 0.362 0.122 0.914
|
94
|
+
TeamsController#progress: 2 0.646 0.497 0.149 1.144
|
95
|
+
RssController#uber: 2 0.008 0.000 0.008 0.008
|
96
|
+
PeopleController#progress: 2 0.415 0.415 0.000 0.830
|
97
|
+
PeopleController#view: 2 0.338 0.149 0.189 0.486
|
98
|
+
|
99
|
+
Average DB Time: 0.402
|
100
|
+
DB Time Std Dev: 0.394
|
101
|
+
|
102
|
+
Slowest Total DB Times:
|
103
|
+
TeamsController#progress took 1.144s
|
104
|
+
ThingsController#view took 0.914s
|
105
|
+
PeopleController#progress took 0.830s
|
106
|
+
PeopleController#view took 0.486s
|
107
|
+
PeopleController#view took 0.189s
|
108
|
+
ThingsController#view took 0.173s
|
109
|
+
TeamsController#progress took 0.149s
|
110
|
+
ThingsController#view took 0.122s
|
111
|
+
RssController#uber took 0.008s
|
112
|
+
RssController#uber took 0.008s
|
113
|
+
|
114
|
+
------------------------------------------------------------------------
|
115
|
+
|
116
|
+
Render Times Summary: Count Avg Std Dev Min Max
|
117
|
+
ALL REQUESTS: 11 0.219 0.253 0.000 0.695
|
118
|
+
|
119
|
+
ThingsController#view: 3 0.270 0.171 0.108 0.506
|
120
|
+
TeamsController#progress: 2 0.000 0.000 0.000 0.000
|
121
|
+
RssController#uber: 2 0.012 0.000 0.012 0.012
|
122
|
+
PeopleController#progress: 2 0.302 0.302 0.000 0.604
|
123
|
+
PeopleController#view: 2 0.487 0.209 0.278 0.695
|
124
|
+
|
125
|
+
Average Render Time: 0.302
|
126
|
+
Render Time Std Dev: 0.251
|
127
|
+
|
128
|
+
Slowest Total Render Times:
|
129
|
+
PeopleController#view took 0.695s
|
130
|
+
PeopleController#progress took 0.604s
|
131
|
+
ThingsController#view took 0.506s
|
132
|
+
PeopleController#view took 0.278s
|
133
|
+
ThingsController#view took 0.197s
|
134
|
+
ThingsController#view took 0.108s
|
135
|
+
RssController#uber took 0.012s
|
136
|
+
RssController#uber took 0.012s
|
137
|
+
TeamsController#progress took 0.000s
|
138
|
+
TeamsController#progress took 0.000s
|
139
|
+
|
140
|
+
== What's missing
|
141
|
+
|
142
|
+
* More reports
|
143
|
+
* Command line arguments including:
|
144
|
+
* Help
|
145
|
+
* What type of log file you've got (if somebody sends patches with tests)
|
146
|
+
* Read from STDIN
|
147
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'hoe'
|
2
|
+
|
3
|
+
$:.unshift './lib'
|
4
|
+
require 'production_log/analyzer'
|
5
|
+
|
6
|
+
Hoe.new 'production_log_analyzer', '2009022403' do |p|
|
7
|
+
p.summary = p.paragraphs_of('README.txt', 1).join ' '
|
8
|
+
p.description = p.paragraphs_of('README.txt', 7).join ' '
|
9
|
+
p.author = 'Eric Hodel'
|
10
|
+
p.email = 'drbrain@segment7.net'
|
11
|
+
p.url = p.paragraphs_of('README.txt', 2).join ' '
|
12
|
+
|
13
|
+
p.rubyforge_name = 'seattlerb'
|
14
|
+
|
15
|
+
p.extra_deps << ['rails_analyzer_tools', '>= 1.4.0']
|
16
|
+
end
|
17
|
+
|
data/bin/action_errors
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
#!/usr/local/bin/ruby -ws
|
2
|
+
|
3
|
+
$h ||= false
|
4
|
+
$r ||= false
|
5
|
+
$o ||= false
|
6
|
+
|
7
|
+
$r = $r ? ($r.to_i rescue false) : false
|
8
|
+
|
9
|
+
if $h then
|
10
|
+
$stderr.puts "Usage: #{$0} [-r=N] LOGFILE"
|
11
|
+
$stderr.puts "\t-r=N\tShow routing errors with N or more occurances"
|
12
|
+
$stderr.puts "\t-o\tShow errors with one occurance"
|
13
|
+
exit
|
14
|
+
end
|
15
|
+
|
16
|
+
errors = {}
|
17
|
+
counts = Hash.new 0
|
18
|
+
|
19
|
+
ARGF.each_line do |line|
|
20
|
+
line =~ /\]: (.*?) (.*)/
|
21
|
+
next if $1.nil?
|
22
|
+
msg = $1
|
23
|
+
trace = $2
|
24
|
+
key = msg.gsub(/\d/, '#')
|
25
|
+
counts[key] += 1
|
26
|
+
next if counts[key] > 1
|
27
|
+
trace = trace.split(' ')[0..-2].map { |l| l.strip }.join("\n\t")
|
28
|
+
error = "#{msg}\n\t#{trace}"
|
29
|
+
errors[key] = error
|
30
|
+
end
|
31
|
+
|
32
|
+
counts.sort_by { |_,c| -c }.each do |key, count|
|
33
|
+
next if count == 1 and not $o
|
34
|
+
error = errors[key]
|
35
|
+
|
36
|
+
if error =~ /^ActionController::RoutingError/ then
|
37
|
+
next unless $r
|
38
|
+
next if $r and count < $r
|
39
|
+
end
|
40
|
+
|
41
|
+
puts "count: #{count}"
|
42
|
+
puts "{{{"
|
43
|
+
puts error
|
44
|
+
puts "}}}"
|
45
|
+
end
|
46
|
+
|
data/bin/action_grep
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
#!/usr/local/bin/ruby -w
|
2
|
+
|
3
|
+
require 'production_log/action_grep'
|
4
|
+
|
5
|
+
action_name = ARGV.shift
|
6
|
+
file_name = ARGV.shift
|
7
|
+
|
8
|
+
if action_name.nil? or file_name.nil? then
|
9
|
+
puts "Usage: #{$0} action_name file_name"
|
10
|
+
exit 1
|
11
|
+
end
|
12
|
+
|
13
|
+
begin
|
14
|
+
ActionGrep.grep action_name, file_name
|
15
|
+
rescue ArgumentError => e
|
16
|
+
puts e
|
17
|
+
exit 1
|
18
|
+
end
|
19
|
+
|
data/bin/pl_analyze
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
#!/bin/env ruby -w
|
2
|
+
|
3
|
+
$:.unshift "#{File.dirname(__FILE__)}/../lib/"
|
4
|
+
require "production_log/analyzer"
|
5
|
+
|
6
|
+
file_name = ARGV.shift
|
7
|
+
|
8
|
+
if file_name.nil? then
|
9
|
+
puts "Usage: #{$0} file_name [-e email_recipient [-s subject]] [count]"
|
10
|
+
exit 1
|
11
|
+
end
|
12
|
+
|
13
|
+
email_recipient = nil
|
14
|
+
subject = nil
|
15
|
+
|
16
|
+
if ARGV.first == '-e' then
|
17
|
+
ARGV.shift # -e
|
18
|
+
email_recipient = ARGV.shift
|
19
|
+
end
|
20
|
+
|
21
|
+
if email_recipient and ARGV.first == '-s' then
|
22
|
+
ARGV.shift # -s
|
23
|
+
subject = ARGV.shift
|
24
|
+
end
|
25
|
+
|
26
|
+
count = ARGV.shift
|
27
|
+
count = count.nil? ? 10 : Integer(count)
|
28
|
+
|
29
|
+
if email_recipient.nil? then
|
30
|
+
analyzer = Analyzer.new file_name
|
31
|
+
analyzer.process
|
32
|
+
puts analyzer.report(count)
|
33
|
+
else
|
34
|
+
Analyzer.email file_name, email_recipient, subject, count
|
35
|
+
end
|
36
|
+
|
@@ -0,0 +1,55 @@
|
|
1
|
+
class PassengerLogPerProc
|
2
|
+
|
3
|
+
cattr_accessor :log_path_prefix
|
4
|
+
|
5
|
+
def self.logger_mutex
|
6
|
+
@@logger_mutex ||= Mutex.new
|
7
|
+
end
|
8
|
+
|
9
|
+
# with log_path_with_prefix like {RAILS_ROOT}/log/passenger/{RAILS_ENV}
|
10
|
+
# you will get logs like {RAILS_ROOT}/log/passenger/{RAILS_ENV}_{Pid}.log
|
11
|
+
def self.enable(log_path_with_prefix)
|
12
|
+
PassengerLogPerProc.log_path_prefix = log_path_with_prefix
|
13
|
+
PhusionPassenger::Rack::RequestHandler.class_eval do
|
14
|
+
cattr_accessor :process_logger
|
15
|
+
|
16
|
+
def process_request_with_extra_logging(env, input, output)
|
17
|
+
unless self.class.process_logger
|
18
|
+
PhusionPassenger::Rack::RequestHandler.process_logger = ActiveSupport::BufferedLogger.new(PassengerLogPerProc.log_path_prefix + "_#{Process.pid}" + ".log")
|
19
|
+
|
20
|
+
PhusionPassenger::Rack::RequestHandler.process_logger.debug("\nPassenger logging started")
|
21
|
+
|
22
|
+
RAILS_DEFAULT_LOGGER.instance_eval do
|
23
|
+
class << self
|
24
|
+
def add(severity, message = nil, progname = nil, &block)
|
25
|
+
to_return = super
|
26
|
+
if to_return
|
27
|
+
Thread.current[:passenger_logs] ||= []
|
28
|
+
Thread.current[:passenger_logs] << [severity, to_return.strip]
|
29
|
+
end
|
30
|
+
to_return
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
Thread.current[:passenger_logs] ||= []
|
37
|
+
start_time = Time.now
|
38
|
+
process_request_without_extra_logging(env, input, output)
|
39
|
+
ensure
|
40
|
+
PassengerLogPerProc.logger_mutex.synchronize do
|
41
|
+
PhusionPassenger::Rack::RequestHandler.process_logger.info("RECEIVE_REQUEST (#{Process.pid}) #{env['REQUEST_METHOD']} #{env['REQUEST_URI']} #{start_time}")
|
42
|
+
Thread.current[:passenger_logs].each do |to_log|
|
43
|
+
PhusionPassenger::Rack::RequestHandler.process_logger.add(to_log[0], to_log[1])
|
44
|
+
end
|
45
|
+
PhusionPassenger::Rack::RequestHandler.process_logger.info("#{Time.now} (#{Process.pid}) RESPONSE_SENT")
|
46
|
+
PhusionPassenger::Rack::RequestHandler.process_logger.info("\n")
|
47
|
+
Thread.current[:passenger_logs] = []
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
alias_method_chain :process_request, :extra_logging
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module ActionGrep; end
|
2
|
+
|
3
|
+
class << ActionGrep
|
4
|
+
|
5
|
+
def grep(action_name, file_name)
|
6
|
+
unless action_name =~ /\A([A-Z][A-Za-z\d]*)(?:#([A-Za-z]\w*))?\Z/ then
|
7
|
+
raise ArgumentError, "Invalid action name #{action_name} expected something like SomeController#action"
|
8
|
+
end
|
9
|
+
|
10
|
+
unless File.file? file_name and File.readable? file_name then
|
11
|
+
raise ArgumentError, "Unable to read #{file_name}"
|
12
|
+
end
|
13
|
+
|
14
|
+
buckets = Hash.new { |h,k| h[k] = [] }
|
15
|
+
comp_count = Hash.new 0
|
16
|
+
|
17
|
+
File.open file_name do |fp|
|
18
|
+
LogParser.detect_mode(fp)
|
19
|
+
fp.each_line do |line|
|
20
|
+
bucket, data = LogParser.extract_bucket_and_data(line)
|
21
|
+
next if !bucket
|
22
|
+
|
23
|
+
buckets[bucket] << line
|
24
|
+
|
25
|
+
case data
|
26
|
+
when /^Start rendering component / then
|
27
|
+
comp_count[bucket] += 1
|
28
|
+
when /^End of component rendering$/ then
|
29
|
+
comp_count[bucket] -= 1
|
30
|
+
when /^Completed/ then
|
31
|
+
next unless comp_count[bucket] == 0
|
32
|
+
action = buckets.delete bucket
|
33
|
+
next unless action.any? { |l| l =~ /Processing #{action_name}/ }
|
34
|
+
puts action.join
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
|