rawk_log 2.0.1 → 2.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/rawk_log +4 -290
- data/lib/rawk_log/command.rb +204 -0
- data/lib/rawk_log/patch_activesupport_bufferedlogger.rb +14 -17
- data/lib/rawk_log/patch_logger.rb +4 -6
- data/lib/rawk_log/stat.rb +101 -0
- data/lib/rawk_log/stat_hash.rb +41 -0
- data/lib/rawk_log/version.rb +1 -1
- data/lib/rawk_log.rb +11 -3
- data/test/empty_test.rb +28 -0
- data/test/examples/empty.log +0 -0
- data/test/examples/rails23.log +3 -0
- data/test/examples/unknown.log +10 -0
- data/test/mixed_rails40_test.rb +7 -1
- data/test/rails23_test.rb +7 -1
- data/test/rails40_test.rb +7 -1
- data/test/self_test.rb +3 -1
- data/test/unknown_test.rb +28 -0
- metadata +12 -1
data/bin/rawk_log
CHANGED
@@ -2,297 +2,11 @@
|
|
2
2
|
##Rail's Analyzer With Klass
|
3
3
|
#run the following to view help:
|
4
4
|
#ruby rawk.rb -?
|
5
|
-
require 'date'
|
6
|
-
class Stat
|
7
|
-
def initialize(key)
|
8
|
-
@key=key
|
9
|
-
@min = nil
|
10
|
-
@max = nil
|
11
|
-
@sum = 0
|
12
|
-
@sum_squares = 0
|
13
|
-
@count = 0
|
14
|
-
@values = []
|
15
|
-
end
|
16
|
-
def add(value)
|
17
|
-
@new_log_format = !value.is_a?(Float)
|
18
|
-
value=1.0*value
|
19
|
-
@count+=1
|
20
|
-
@min = value unless @min
|
21
|
-
@min = value if value<@min
|
22
|
-
@max = value unless @max
|
23
|
-
@max = value if value>@max
|
24
|
-
@sum += value
|
25
|
-
@sum_squares += value*value
|
26
|
-
@values << value
|
27
|
-
end
|
28
|
-
def key
|
29
|
-
@key
|
30
|
-
end
|
31
|
-
def count
|
32
|
-
@count
|
33
|
-
end
|
34
|
-
def sum
|
35
|
-
@sum
|
36
|
-
end
|
37
|
-
def min
|
38
|
-
@min
|
39
|
-
end
|
40
|
-
def max
|
41
|
-
@max
|
42
|
-
end
|
43
|
-
def average
|
44
|
-
@sum/@count
|
45
|
-
end
|
46
|
-
def median
|
47
|
-
return nil unless @values
|
48
|
-
l = @values.length
|
49
|
-
return nil unless l>0
|
50
|
-
@values.sort!
|
51
|
-
return (@values[l/2-1]+@values[l/2])/2 if l%2==0
|
52
|
-
@values[(l+1)/2-1]
|
53
|
-
end
|
54
|
-
def standard_deviation
|
55
|
-
return 0 if @count<=1
|
56
|
-
Math.sqrt((@sum_squares - (@sum*@sum/@count))/ (@count) )
|
57
|
-
end
|
58
|
-
def to_s
|
59
|
-
if @new_log_format
|
60
|
-
sprintf("%-55s %6d %7.2f %7d %7d %7d %7d %7d",key,count,(sum.to_f/1000),max,median,average,min,standard_deviation)
|
61
|
-
else
|
62
|
-
sprintf("%-55s %6d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f",key,count,sum,max,median,average,min,standard_deviation)
|
63
|
-
end
|
64
|
-
end
|
65
5
|
|
66
|
-
|
67
|
-
stat = Stat.new(30)
|
68
|
-
stat.add(5)
|
69
|
-
stat.add(6)
|
70
|
-
stat.add(8)
|
71
|
-
stat.add(9)
|
72
|
-
messages = [ 7==stat.median ? "median Success" : "median Failure" ]
|
73
|
-
messages <<= (7==stat.average ? "average Success" : "average Failure")
|
74
|
-
messages <<= (158==(stat.standard_deviation*100).round ? "std Success" : "std Failure")
|
75
|
-
puts messages.join("\n")
|
76
|
-
exit (messages.select{|m| m =~ /Failure/}.size)
|
77
|
-
end
|
6
|
+
$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
|
78
7
|
|
79
|
-
|
80
|
-
class StatHash
|
81
|
-
def initialize
|
82
|
-
@stats = Hash.new
|
83
|
-
end
|
84
|
-
def add(key,time)
|
85
|
-
stat = @stats[key] || (@stats[key] = Stat.new(key))
|
86
|
-
stat.add(time)
|
87
|
-
end
|
88
|
-
def print(args={:sort_by=>'key',:ascending=>true,:limit=>nil})
|
89
|
-
values = @stats.values
|
90
|
-
order = (args[:ascending] || args[:ascending].nil?) ? 1 : -1
|
91
|
-
values.sort! {|a,b|
|
92
|
-
as = a.send(args[:sort_by])
|
93
|
-
bs = b.send(args[:sort_by])
|
94
|
-
(as && bs) ? order*(as<=>bs) : 0
|
95
|
-
}
|
96
|
-
#values.sort! {|a,b| a.key<=>b.key}
|
97
|
-
limit = args[:limit]
|
98
|
-
for stat in values
|
99
|
-
break if limit && limit<=0
|
100
|
-
puts stat.to_s
|
101
|
-
limit-=1 if limit
|
102
|
-
end
|
103
|
-
end
|
104
|
-
end
|
8
|
+
require 'rawk_log/command'
|
105
9
|
|
106
|
-
|
107
|
-
|
108
|
-
HEADER = "Request Count Sum Max Median Avg Min Std"
|
109
|
-
HEADER_NEW_LOG_FORMAT = "Request Count Sum(s) Max Median Avg Min Std"
|
110
|
-
HELP = "\nRAWK - Rail's Analyzer With Klass v#{VERSION}\n"+
|
111
|
-
"Created by Chris Hobbs of Spongecell, LLC\n"+
|
112
|
-
"This tool gives statistics for Ruby on Rails log files. The times for each request are grouped and totals are displayed. "+
|
113
|
-
"If process ids are present in the log files then requests are sorted by ActionController actions otherwise requests are grouped by url. "+
|
114
|
-
"By default total request times are used for comparison but database time or render time can be used by specifying the correct flag. "+
|
115
|
-
"The log file is read from standard input unless the -f flag is specified.\n\n"+
|
116
|
-
"The options are as follows:\n\n"+
|
117
|
-
" -? Display this help.\n\n"+
|
118
|
-
" -d Use DB times as data points. These times are found after 'DB:' in the log file. This overrides the default behavior of using the total request time.\n\n"+
|
119
|
-
" -f <filename> Use the specified file instead of standard input.\n\n"+
|
120
|
-
" -h Display this help.\n\n"+
|
121
|
-
" -r Use Render times as data points. These times are found after 'Rendering:' in the log file. This overrides the default behavior of using the total request time.\n\n"+
|
122
|
-
" -s <count> Display <count> results in each group of data.\n\n"+
|
123
|
-
" -t Test\n\n"+
|
124
|
-
" -u Group requests by url instead of the controller and action used. This is the default behavior if there is are no process ids in the log file.\n\n"+
|
125
|
-
" -w <count> Display the top <count> worst requests.\n\n"+
|
126
|
-
" -x <date> Date (inclusive) to start parsing in 'yyyy-mm-dd' format.\n\n"+
|
127
|
-
" -y <date> Date (inclusive) to stop parsing in 'yyyy-mm-dd' format.\n\n"+
|
128
|
-
"To include process ids in your log file, add this to application's Gemfile:\n\n"+
|
129
|
-
" gem 'rawk_log'\n\n"+
|
130
|
-
"This software is Beerware, if you like it, buy yourself a beer or something nicer ;)\n"+
|
131
|
-
"\n"+
|
132
|
-
"Example usage:\n"+
|
133
|
-
" rawk_log log/production.log\n"
|
134
|
-
|
135
|
-
def initialize
|
136
|
-
@start_time = Time.now
|
137
|
-
build_arg_hash
|
138
|
-
if @arg_hash.keys.include?("?") || @arg_hash.keys.include?("h")
|
139
|
-
puts HELP
|
140
|
-
elsif @arg_hash.keys.include?("t")
|
141
|
-
Stat.test
|
142
|
-
else
|
143
|
-
init_args
|
144
|
-
build_stats
|
145
|
-
print_stats
|
146
|
-
end
|
147
|
-
end
|
148
|
-
def build_arg_hash
|
149
|
-
@arg_hash = Hash.new
|
150
|
-
last_key=nil
|
151
|
-
for a in $*
|
152
|
-
if a.index("-")==0 && a.length>1
|
153
|
-
a[1,1000].scan(/[a-z]|\?/).each {|c| @arg_hash[last_key=c]=nil}
|
154
|
-
@arg_hash[last_key] = a[/\d+/] if last_key
|
155
|
-
elsif a.index("-")!=0 && last_key
|
156
|
-
@arg_hash[last_key] = a
|
157
|
-
end
|
158
|
-
end
|
159
|
-
#$* = [$*[0]]
|
160
|
-
end
|
161
|
-
def init_args
|
162
|
-
@sorted_limit=20
|
163
|
-
@worst_request_length=20
|
164
|
-
@force_url_use = false
|
165
|
-
@db_time = false
|
166
|
-
@render_time = false
|
167
|
-
@input = $stdin
|
168
|
-
keys = @arg_hash.keys
|
169
|
-
@force_url_use = keys.include?("u")
|
170
|
-
@db_time = keys.include?("d")
|
171
|
-
@render_time = keys.include?("r")
|
172
|
-
@worst_request_length=(@arg_hash["w"].to_i) if @arg_hash["w"]
|
173
|
-
@sorted_limit = @arg_hash["s"].to_i if @arg_hash["s"]
|
174
|
-
@input = File.new(@arg_hash["f"]) if @arg_hash["f"]
|
175
|
-
@from =(Date.parse(@arg_hash["x"])) if @arg_hash["x"]
|
176
|
-
@to =(Date.parse(@arg_hash["y"])) if @arg_hash["y"]
|
177
|
-
end
|
178
|
-
|
179
|
-
def is_id?(word)
|
180
|
-
word =~ /^((\d+)|([\dA-F\-]{36}))$/i
|
181
|
-
end
|
182
|
-
|
183
|
-
def build_stats
|
184
|
-
@stat_hash = StatHash.new
|
185
|
-
@total_stat = Stat.new("All Requests")
|
186
|
-
@worst_requests = []
|
187
|
-
last_actions = Hash.new
|
188
|
-
while @input.gets
|
189
|
-
if $_.index("Processing ")==0
|
190
|
-
action = $_.split[1]
|
191
|
-
pid = $_[/\(pid\:\d+\)/]
|
192
|
-
date = Date.parse($_[/(?:19|20)[0-9]{2}-(?:0[1-9]|1[012])-(?:0[1-9]|[12][0-9]|3[01])/])
|
193
|
-
datetime = $_[/(?:19|20)[0-9]{2}-(?:0[1-9]|1[012])-(?:0[1-9]|[12][0-9]|3[01]) (?:[0-1][0-9]|2[0-3]):(?:[0-5][0-9]|60):(?:[0-5][0-9]|60)/]
|
194
|
-
last_actions[pid]=action if pid
|
195
|
-
next
|
196
|
-
end
|
197
|
-
next unless $_.index("Completed in")==0
|
198
|
-
pid = key = nil
|
199
|
-
#get the pid unless we are forcing url tracking
|
200
|
-
pid = $_[/\(pid\:\d+\)/] if !@force_url_use
|
201
|
-
key = last_actions[pid] if pid
|
202
|
-
time = 0.0
|
203
|
-
|
204
|
-
# Old: Completed in 0.45141 (2 reqs/sec) | Rendering: 0.25965 (57%) | DB: 0.06300 (13%) | 200 OK [http://localhost/jury/proposal/312]
|
205
|
-
# New: Completed in 100ms (View: 40, DB: 4)
|
206
|
-
unless defined? @new_log_format
|
207
|
-
@new_log_format = $_ =~ /Completed in \d+ms/
|
208
|
-
end
|
209
|
-
|
210
|
-
if @new_log_format
|
211
|
-
if @db_time
|
212
|
-
time_string = $_[/DB: \d+/]
|
213
|
-
elsif @render_time
|
214
|
-
time_string = $_[/View: \d+/]
|
215
|
-
else
|
216
|
-
time_string = $_[/Completed in \d+ms/]
|
217
|
-
end
|
218
|
-
time_string = time_string[/\d+/] if time_string
|
219
|
-
time = time_string.to_i if time_string
|
220
|
-
else
|
221
|
-
if @db_time
|
222
|
-
time_string = $_[/DB: \d+\.\d+/]
|
223
|
-
elsif @render_time
|
224
|
-
time_string = $_[/Rendering: \d+\.\d+/]
|
225
|
-
else
|
226
|
-
time_string = $_[/Completed in \d+\.\d+/]
|
227
|
-
end
|
228
|
-
time_string = time_string[/\d+\.\d+/] if time_string
|
229
|
-
time = time_string.to_f if time_string
|
230
|
-
end
|
231
|
-
|
232
|
-
|
233
|
-
#if pids are not specified then we use the url for hashing
|
234
|
-
#the below regexp turns "[http://spongecell.com/calendar/view/bob]" to "/calendar/view"
|
235
|
-
unless key
|
10
|
+
cmd = RawkLog::Command.new($*)
|
11
|
+
cmd.run
|
236
12
|
|
237
|
-
key = if @force_url_use
|
238
|
-
($_[/\[\S+\]/].gsub(/\S+\/\/(\w|\.)*/,''))[/[^\?\]]*/]
|
239
|
-
else
|
240
|
-
data = $_[/\[\S+\]/].gsub(/\S+\/\/(\w|\.)*/,'')
|
241
|
-
s = data.gsub(/(\?.*)|\]$/,'').split("/")
|
242
|
-
|
243
|
-
keywords = s.inject([]) do |keywords, word|
|
244
|
-
if is_id?(word.to_s)
|
245
|
-
keywords << '{ID}'
|
246
|
-
elsif !word.to_s.empty?
|
247
|
-
keywords << word.to_s
|
248
|
-
end
|
249
|
-
keywords
|
250
|
-
end
|
251
|
-
k = "/#{keywords.join("/")}"
|
252
|
-
end
|
253
|
-
end
|
254
|
-
|
255
|
-
if (@from.nil? or @from <= date) and (@to.nil? or @to >= date) # date criteria here
|
256
|
-
@stat_hash.add(key,time)
|
257
|
-
@total_stat.add(time)
|
258
|
-
if @worst_requests.length<@worst_request_length || @worst_requests[@worst_request_length-1][0]<time
|
259
|
-
@worst_requests << [time,%Q(#{datetime} #{$_})]
|
260
|
-
@worst_requests.sort! {|a,b| (b[0] && a[0]) ? b[0]<=>a[0] : 0}
|
261
|
-
@worst_requests=@worst_requests[0,@worst_request_length]
|
262
|
-
end
|
263
|
-
end
|
264
|
-
end
|
265
|
-
end
|
266
|
-
def print_stats
|
267
|
-
@header ||= @new_log_format ? HEADER_NEW_LOG_FORMAT : HEADER
|
268
|
-
puts "Printing report for #{@db_time ? 'DB' : @render_time ? 'render' : 'total'} request times#{@from ? %Q( from #{@from.to_s}) : ""}#{@to ? %Q( through #{@to.to_s}) : ""}"
|
269
|
-
puts "--------"
|
270
|
-
puts @header
|
271
|
-
puts @total_stat.to_s
|
272
|
-
puts "--------"
|
273
|
-
@stat_hash.print()
|
274
|
-
puts "\nTop #{@sorted_limit} by Count"
|
275
|
-
puts @header
|
276
|
-
@stat_hash.print(:sort_by=>"count",:limit=>@sorted_limit,:ascending=>false)
|
277
|
-
puts "\nTop #{@sorted_limit} by Sum of Time"
|
278
|
-
puts @header
|
279
|
-
@stat_hash.print(:sort_by=>"sum",:limit=>@sorted_limit,:ascending=>false)
|
280
|
-
puts "\nTop #{@sorted_limit} Greatest Max"
|
281
|
-
puts @header
|
282
|
-
@stat_hash.print(:sort_by=>"max",:limit=>@sorted_limit,:ascending=>false)
|
283
|
-
puts "\nTop #{@sorted_limit} Least Min"
|
284
|
-
puts @header
|
285
|
-
@stat_hash.print(:sort_by=>"min",:limit=>@sorted_limit)
|
286
|
-
puts "\nTop #{@sorted_limit} Greatest Median"
|
287
|
-
puts @header
|
288
|
-
@stat_hash.print(:sort_by=>"median",:limit=>@sorted_limit,:ascending=>false)
|
289
|
-
puts "\nTop #{@sorted_limit} Greatest Standard Deviation"
|
290
|
-
puts @header
|
291
|
-
@stat_hash.print(:sort_by=>"standard_deviation",:limit=>@sorted_limit,:ascending=>false)
|
292
|
-
puts "\nWorst Requests"
|
293
|
-
@worst_requests.each {|w| puts w[1].to_s}
|
294
|
-
puts "\nCompleted report in #{(Time.now.to_i-@start_time.to_i)/60.0} minutes -- spongecell"
|
295
|
-
end
|
296
|
-
end
|
297
|
-
|
298
|
-
Rawk.new
|
@@ -0,0 +1,204 @@
|
|
1
|
+
require 'date'
|
2
|
+
require 'rawk_log/stat'
|
3
|
+
require 'rawk_log/stat_hash'
|
4
|
+
|
5
|
+
module RawkLog
|
6
|
+
class Command
|
7
|
+
HELP = "\nRAWK - Rail's Analyzer With Klass\n"+
|
8
|
+
"Created by Chris Hobbs of Spongecell, LLC\n"+
|
9
|
+
"This tool gives statistics for Ruby on Rails log files. The times for each request are grouped and totals are displayed. "+
|
10
|
+
"If process ids are present in the log files then requests are sorted by ActionController actions otherwise requests are grouped by url. "+
|
11
|
+
"By default total request times are used for comparison but database time or render time can be used by specifying the correct flag. "+
|
12
|
+
"The log file is read from standard input unless the -f flag is specified.\n\n"+
|
13
|
+
"The options are as follows:\n\n"+
|
14
|
+
" -? Display this help.\n\n"+
|
15
|
+
" -d Use DB times as data points. These times are found after 'DB:' in the log file. This overrides the default behavior of using the total request time.\n\n"+
|
16
|
+
" -f <filename> Use the specified file instead of standard input.\n\n"+
|
17
|
+
" -h Display this help.\n\n"+
|
18
|
+
" -r Use Render times as data points. These times are found after 'Rendering:' in the log file. This overrides the default behavior of using the total request time.\n\n"+
|
19
|
+
" -s <count> Display <count> results in each group of data.\n\n"+
|
20
|
+
" -t Test\n\n"+
|
21
|
+
" -u Group requests by url instead of the controller and action used. This is the default behavior if there is are no process ids in the log file.\n\n"+
|
22
|
+
" -w <count> Display the top <count> worst requests.\n\n"+
|
23
|
+
" -x <date> Date (inclusive) to start parsing in 'yyyy-mm-dd' format.\n\n"+
|
24
|
+
" -y <date> Date (inclusive) to stop parsing in 'yyyy-mm-dd' format.\n\n"+
|
25
|
+
"To include process ids in your log file, add this to application's Gemfile:\n\n"+
|
26
|
+
" gem 'rawk_log'\n\n"+
|
27
|
+
"This software is Beerware, if you like it, buy yourself a beer or something nicer ;)\n"+
|
28
|
+
"\n"+
|
29
|
+
"Example usage:\n"+
|
30
|
+
" rawk_log log/production.log\n"
|
31
|
+
|
32
|
+
def initialize(args)
|
33
|
+
@start_time = Time.now
|
34
|
+
build_arg_hash(args)
|
35
|
+
end
|
36
|
+
|
37
|
+
def run
|
38
|
+
if @arg_hash.keys.include?("?") || @arg_hash.keys.include?("h")
|
39
|
+
puts HELP
|
40
|
+
elsif @arg_hash.keys.include?("t")
|
41
|
+
Stat.test
|
42
|
+
else
|
43
|
+
init_args
|
44
|
+
build_stats
|
45
|
+
print_stats
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def build_arg_hash(args)
|
50
|
+
@arg_hash = Hash.new
|
51
|
+
last_key=nil
|
52
|
+
for a in args
|
53
|
+
if a.index("-")==0 && a.length>1
|
54
|
+
a[1,1000].scan(/[a-z]|\?/).each {|c| @arg_hash[last_key=c]=nil}
|
55
|
+
@arg_hash[last_key] = a[/\d+/] if last_key
|
56
|
+
elsif a.index("-")!=0 && last_key
|
57
|
+
@arg_hash[last_key] = a
|
58
|
+
end
|
59
|
+
end
|
60
|
+
#$* = [$*[0]]
|
61
|
+
end
|
62
|
+
|
63
|
+
def init_args
|
64
|
+
@sorted_limit=20
|
65
|
+
@worst_request_length=20
|
66
|
+
@force_url_use = false
|
67
|
+
@db_time = false
|
68
|
+
@render_time = false
|
69
|
+
@input = $stdin
|
70
|
+
keys = @arg_hash.keys
|
71
|
+
@force_url_use = keys.include?("u")
|
72
|
+
@db_time = keys.include?("d")
|
73
|
+
@render_time = keys.include?("r")
|
74
|
+
@worst_request_length=(@arg_hash["w"].to_i) if @arg_hash["w"]
|
75
|
+
@sorted_limit = @arg_hash["s"].to_i if @arg_hash["s"]
|
76
|
+
@input = File.new(@arg_hash["f"]) if @arg_hash["f"]
|
77
|
+
@from =(Date.parse(@arg_hash["x"])) if @arg_hash["x"]
|
78
|
+
@to =(Date.parse(@arg_hash["y"])) if @arg_hash["y"]
|
79
|
+
end
|
80
|
+
|
81
|
+
def is_id?(word)
|
82
|
+
word =~ /^((\d+)|([\dA-F\-]{36}))$/i
|
83
|
+
end
|
84
|
+
|
85
|
+
def build_stats
|
86
|
+
@stat_hash = StatHash.new
|
87
|
+
@total_stat = Stat.new("All Requests")
|
88
|
+
@worst_requests = []
|
89
|
+
last_actions = Hash.new
|
90
|
+
last_date = Date.civil
|
91
|
+
while @input.gets
|
92
|
+
if $_.index("Processing ")==0
|
93
|
+
action = $_.split[1]
|
94
|
+
pid = $_[/\(pid\:\d+\)/]
|
95
|
+
date_string = $_[/(?:19|20)[0-9]{2}-(?:0[1-9]|1[012])-(?:0[1-9]|[12][0-9]|3[01])/]
|
96
|
+
date = date_string ? Date.parse(date_string) : last_date
|
97
|
+
last_date = date
|
98
|
+
datetime = $_[/(?:19|20)[0-9]{2}-(?:0[1-9]|1[012])-(?:0[1-9]|[12][0-9]|3[01]) (?:[0-1][0-9]|2[0-3]):(?:[0-5][0-9]|60):(?:[0-5][0-9]|60)/].to_s
|
99
|
+
last_actions[pid]=action if pid
|
100
|
+
next
|
101
|
+
end
|
102
|
+
next unless $_.index("Completed in")==0
|
103
|
+
pid = key = nil
|
104
|
+
#get the pid unless we are forcing url tracking
|
105
|
+
pid = $_[/\(pid\:\d+\)/] if !@force_url_use
|
106
|
+
key = last_actions[pid] if pid
|
107
|
+
time = 0.0
|
108
|
+
|
109
|
+
# Old: Completed in 0.45141 (2 reqs/sec) | Rendering: 0.25965 (57%) | DB: 0.06300 (13%) | 200 OK [http://localhost/jury/proposal/312]
|
110
|
+
# New: Completed in 100ms (View: 40, DB: 4)
|
111
|
+
unless defined? @new_log_format
|
112
|
+
@new_log_format = $_ =~ /Completed in \d+ms/
|
113
|
+
end
|
114
|
+
|
115
|
+
if @new_log_format
|
116
|
+
if @db_time
|
117
|
+
time_string = $_[/DB: \d+/]
|
118
|
+
elsif @render_time
|
119
|
+
time_string = $_[/View: \d+/]
|
120
|
+
else
|
121
|
+
time_string = $_[/Completed in \d+ms/]
|
122
|
+
end
|
123
|
+
time_string = time_string[/\d+/] if time_string
|
124
|
+
time = time_string.to_i if time_string
|
125
|
+
else
|
126
|
+
if @db_time
|
127
|
+
time_string = $_[/DB: \d+\.\d+/]
|
128
|
+
elsif @render_time
|
129
|
+
time_string = $_[/Rendering: \d+\.\d+/]
|
130
|
+
else
|
131
|
+
time_string = $_[/Completed in \d+\.\d+/]
|
132
|
+
end
|
133
|
+
time_string = time_string[/\d+\.\d+/] if time_string
|
134
|
+
time = time_string.to_f if time_string
|
135
|
+
end
|
136
|
+
|
137
|
+
|
138
|
+
#if pids are not specified then we use the url for hashing
|
139
|
+
#the below regexp turns "[http://spongecell.com/calendar/view/bob]" to "/calendar/view"
|
140
|
+
unless key
|
141
|
+
|
142
|
+
key = if @force_url_use
|
143
|
+
($_[/\[[^\]]+\]/].gsub(/\S+\/\/(\w|\.)*/,''))[/[^\?\]]*/]
|
144
|
+
else
|
145
|
+
data = $_[/\[[^\]]+\]/].gsub(/\S+\/\/(\w|\.)*/,'')
|
146
|
+
s = data.gsub(/(\?.*)|\]$/,'').split("/")
|
147
|
+
|
148
|
+
keywords = s.inject([]) do |keywords, word|
|
149
|
+
if is_id?(word.to_s)
|
150
|
+
keywords << '{ID}'
|
151
|
+
elsif !word.to_s.empty?
|
152
|
+
keywords << word.to_s
|
153
|
+
end
|
154
|
+
keywords
|
155
|
+
end
|
156
|
+
k = "/#{keywords.join("/")}"
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
if (@from.nil? or @from <= date) and (@to.nil? or @to >= date) # date criteria here
|
161
|
+
@stat_hash.add(key,time)
|
162
|
+
@total_stat.add(time)
|
163
|
+
if @worst_requests.length<@worst_request_length || @worst_requests[@worst_request_length-1][0]<time
|
164
|
+
@worst_requests << [time,%Q(#{datetime} #{$_})]
|
165
|
+
@worst_requests.sort! {|a,b| (b[0] && a[0]) ? b[0]<=>a[0] : 0}
|
166
|
+
@worst_requests=@worst_requests[0,@worst_request_length]
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
def print_stats
|
172
|
+
title = "Log Analysis of #{@db_time ? 'DB' : @render_time ? 'render' : 'total'} request times#{@from ? %Q( from #{@from.to_s}) : ""}#{@to ? %Q( through #{@to.to_s}) : ""}"
|
173
|
+
puts title
|
174
|
+
puts "=" * title.size
|
175
|
+
puts ""
|
176
|
+
label_size = @stat_hash.print()
|
177
|
+
if @stat_hash.empty?
|
178
|
+
puts @total_stat.header(label_size)
|
179
|
+
else
|
180
|
+
puts "-" * label_size
|
181
|
+
end
|
182
|
+
puts @total_stat.to_s(label_size)
|
183
|
+
if not @stat_hash.empty?
|
184
|
+
puts "\n\nTop #{@sorted_limit} by Count"
|
185
|
+
@stat_hash.print(:sort_by=>"count",:limit=>@sorted_limit,:ascending=>false)
|
186
|
+
puts "\n\nTop #{@sorted_limit} by Sum of Time"
|
187
|
+
@stat_hash.print(:sort_by=>"sum",:limit=>@sorted_limit,:ascending=>false)
|
188
|
+
puts "\n\nTop #{@sorted_limit} Greatest Max"
|
189
|
+
@stat_hash.print(:sort_by=>"max",:limit=>@sorted_limit,:ascending=>false)
|
190
|
+
puts "\n\nTop #{@sorted_limit} Greatest Median"
|
191
|
+
@stat_hash.print(:sort_by=>"median",:limit=>@sorted_limit,:ascending=>false)
|
192
|
+
puts "\n\nTop #{@sorted_limit} Greatest Avg"
|
193
|
+
@stat_hash.print(:sort_by=>"average",:limit=>@sorted_limit,:ascending=>false)
|
194
|
+
puts "\n\nTop #{@sorted_limit} Least Min"
|
195
|
+
@stat_hash.print(:sort_by=>"min",:limit=>@sorted_limit)
|
196
|
+
puts "\n\nTop #{@sorted_limit} Greatest Standard Deviation"
|
197
|
+
@stat_hash.print(:sort_by=>"standard_deviation",:limit=>@sorted_limit,:ascending=>false)
|
198
|
+
puts "\n\nTop #{@worst_request_length} Worst Requests"
|
199
|
+
@worst_requests.each {|w| puts w[1].to_s}
|
200
|
+
end
|
201
|
+
puts "\n\nCompleted report in %1.2f minutes" % ((Time.now.to_i-@start_time.to_i)/60.0)
|
202
|
+
end
|
203
|
+
end
|
204
|
+
end
|
@@ -1,22 +1,19 @@
|
|
1
1
|
|
2
|
-
|
2
|
+
module ActiveSupport
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
auto_flush
|
17
|
-
message
|
18
|
-
end
|
4
|
+
# Format the buffered logger with timestamp/severity info.
|
5
|
+
class BufferedLogger
|
6
|
+
def add(severity, message = nil, progname = nil, &block)
|
7
|
+
return if @level > severity
|
8
|
+
message = (message || (block && block.call) || progname).to_s
|
9
|
+
# If a newline is necessary then create a new message ending with a newline.
|
10
|
+
# Ensures that the original message is not mutated.
|
11
|
+
message = "#{message}\n" unless message[-1] == ?\n
|
12
|
+
message = message.gsub(/\n/," (pid:#{$$})\n")
|
13
|
+
buffer << message
|
14
|
+
auto_flush
|
15
|
+
message
|
19
16
|
end
|
20
17
|
end
|
21
|
-
|
22
18
|
end
|
19
|
+
|
@@ -0,0 +1,101 @@
|
|
1
|
+
module RawkLog
|
2
|
+
|
3
|
+
class Stat
|
4
|
+
|
5
|
+
HEADER = "Count Sum Max Median Avg Min Std"
|
6
|
+
HEADER_NEW_LOG_FORMAT = "Count Sum(s) Max Median Avg Min Std"
|
7
|
+
|
8
|
+
def initialize(key)
|
9
|
+
@key=key
|
10
|
+
@min = nil
|
11
|
+
@max = nil
|
12
|
+
@sum = 0
|
13
|
+
@sum_squares = 0
|
14
|
+
@count = 0
|
15
|
+
@values = []
|
16
|
+
end
|
17
|
+
|
18
|
+
def add(value)
|
19
|
+
@new_log_format = !value.is_a?(Float)
|
20
|
+
value=1.0*value
|
21
|
+
@count+=1
|
22
|
+
@min = value unless @min
|
23
|
+
@min = value if value<@min
|
24
|
+
@max = value unless @max
|
25
|
+
@max = value if value>@max
|
26
|
+
@sum += value
|
27
|
+
@sum_squares += value*value
|
28
|
+
@values << value
|
29
|
+
end
|
30
|
+
|
31
|
+
def header(label_size = 55)
|
32
|
+
header = @new_log_format ? HEADER_NEW_LOG_FORMAT : HEADER
|
33
|
+
sprintf "%*s %s" % [-label_size, "Request", header]
|
34
|
+
end
|
35
|
+
|
36
|
+
def key
|
37
|
+
@key
|
38
|
+
end
|
39
|
+
|
40
|
+
def count
|
41
|
+
@count
|
42
|
+
end
|
43
|
+
|
44
|
+
def sum
|
45
|
+
@sum
|
46
|
+
end
|
47
|
+
|
48
|
+
def min
|
49
|
+
@min
|
50
|
+
end
|
51
|
+
|
52
|
+
def max
|
53
|
+
@max
|
54
|
+
end
|
55
|
+
|
56
|
+
def average
|
57
|
+
@count > 0 ? @sum/@count : @sum
|
58
|
+
end
|
59
|
+
|
60
|
+
def median
|
61
|
+
return nil unless @values
|
62
|
+
l = @values.length
|
63
|
+
return nil unless l>0
|
64
|
+
@values.sort!
|
65
|
+
return (@values[l/2-1]+@values[l/2])/2 if l%2==0
|
66
|
+
@values[(l+1)/2-1]
|
67
|
+
end
|
68
|
+
|
69
|
+
def standard_deviation
|
70
|
+
return 0 if @count<=1
|
71
|
+
Math.sqrt((@sum_squares - (@sum*@sum/@count))/ (@count) )
|
72
|
+
end
|
73
|
+
|
74
|
+
def to_s(label_size = 55)
|
75
|
+
if count > 0
|
76
|
+
if @new_log_format
|
77
|
+
sprintf("%*s %6d %7.2f %7d %7d %7d %7d %7d",-label_size, key,count,(sum.to_f/1000),max,median,average,min,standard_deviation)
|
78
|
+
else
|
79
|
+
sprintf("%*s %6d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f",-label_size,key,count,sum,max,median,average,min,standard_deviation)
|
80
|
+
end
|
81
|
+
else
|
82
|
+
sprintf("%*s %6d",-label_size,key,0)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def self.test
|
87
|
+
stat = Stat.new(30)
|
88
|
+
stat.add(5)
|
89
|
+
stat.add(6)
|
90
|
+
stat.add(8)
|
91
|
+
stat.add(9)
|
92
|
+
results = [ 7==stat.median ? "median Success" : "median Failure" ]
|
93
|
+
results <<= (7==stat.average ? "average Success" : "average Failure")
|
94
|
+
results <<= (158==(stat.standard_deviation*100).round ? "std Success" : "std Failure")
|
95
|
+
puts results.join("\n")
|
96
|
+
exit (results.select{|m| m =~ /Failure/}.size)
|
97
|
+
end
|
98
|
+
|
99
|
+
end
|
100
|
+
|
101
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'rawk_log/stat'
|
2
|
+
|
3
|
+
module RawkLog
|
4
|
+
class StatHash
|
5
|
+
def initialize
|
6
|
+
@stats = Hash.new
|
7
|
+
end
|
8
|
+
|
9
|
+
def empty?
|
10
|
+
@stats.empty?
|
11
|
+
end
|
12
|
+
|
13
|
+
def add(key,time)
|
14
|
+
stat = @stats[key] || (@stats[key] = RawkLog::Stat.new(key))
|
15
|
+
stat.add(time)
|
16
|
+
end
|
17
|
+
|
18
|
+
def print(args={:sort_by=>'key',:ascending=>true,:limit=>nil})
|
19
|
+
values = @stats.values
|
20
|
+
return 20 if values.empty?
|
21
|
+
order = (args[:ascending] || args[:ascending].nil?) ? 1 : -1
|
22
|
+
values.sort! {|a,b|
|
23
|
+
as = a.send(args[:sort_by])
|
24
|
+
bs = b.send(args[:sort_by])
|
25
|
+
(as && bs) ? order*(as<=>bs) : 0
|
26
|
+
}
|
27
|
+
#values.sort! {|a,b| a.key<=>b.key}
|
28
|
+
limit = args[:limit]
|
29
|
+
if limit
|
30
|
+
values = values[0,limit]
|
31
|
+
end
|
32
|
+
@label_size = values.collect{|v| v.key.size }.max
|
33
|
+
puts values[0].header(@label_size)
|
34
|
+
for stat in values
|
35
|
+
puts stat.to_s(@label_size)
|
36
|
+
end
|
37
|
+
@label_size
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
data/lib/rawk_log/version.rb
CHANGED
data/lib/rawk_log.rb
CHANGED
@@ -1,7 +1,15 @@
|
|
1
1
|
require "rawk_log/version"
|
2
|
-
|
3
|
-
|
2
|
+
|
3
|
+
# Don't load automatically
|
4
|
+
#require "rawk_log/command"
|
5
|
+
|
6
|
+
if defined?(ActiveSupport::BufferedLogger)
|
7
|
+
require "rawk_log/patch_logger"
|
8
|
+
end
|
9
|
+
if defined?(ActiveSupport::BufferedLogger)
|
10
|
+
require "rawk_log/patch_activesupport_bufferedlogger"
|
11
|
+
end
|
4
12
|
|
5
13
|
module RawkLog
|
6
|
-
#
|
14
|
+
# Everything loaded in requires above
|
7
15
|
end
|
data/test/empty_test.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'test/unit'
|
2
|
+
|
3
|
+
class EmptyTest < Test::Unit::TestCase
|
4
|
+
|
5
|
+
def setup
|
6
|
+
bin = File.join(File.dirname(__FILE__), '..', 'bin')
|
7
|
+
examples = File.join(File.dirname(__FILE__), 'examples')
|
8
|
+
@output = `ruby #{bin}/rawk_log -f #{examples}/empty.log`
|
9
|
+
@exit_status = $?.to_i
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_outputs_header
|
13
|
+
assert_match(/^Request +Count +Sum +Max +Median +Avg +Min +Std$/, @output)
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_lists_zero_entries
|
17
|
+
assert_match(/^All Requests +0$/, @output)
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_no_top_lists
|
21
|
+
assert_no_match(/^Top /, @output)
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_exit_status
|
25
|
+
assert_equal(0, @exit_status)
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
File without changes
|
data/test/examples/rails23.log
CHANGED
@@ -25,3 +25,6 @@ Rendering template within layouts/items
|
|
25
25
|
Rendering items/index
|
26
26
|
Completed in 5ms (View: 2, DB: 0) | 200 OK [http://0.0.0.0/items]
|
27
27
|
|
28
|
+
Processing DocumentController#download (for 12.13.14.15 at 2012-12-13 11:49:03) [GET]
|
29
|
+
Completed in 936ms (View: 1, DB: 31) | 200 OK [http://www.example.com/document/download/96164/Form 6 - Occupancy permit-2034-20156 / Unit 1.docx]
|
30
|
+
|
@@ -0,0 +1,10 @@
|
|
1
|
+
# From http://railscasts.com/episodes/56-the-logger-revised?view=asciicast
|
2
|
+
|
3
|
+
Started GET "/" for 127.0.0.1 at 2012-04-23 20:59:50 +0100
|
4
|
+
Processing by ArticlesController#index as HTML
|
5
|
+
Article Load (0.1ms) SELECT "articles".* FROM "articles"
|
6
|
+
Articles Count: 3
|
7
|
+
Rendered articles/index.html.erb within layouts/application (2.8ms)
|
8
|
+
Completed 200 OK in 18ms (Views: 8.4ms | ActiveRecord: 0.7ms)
|
9
|
+
[2012-04-23 20:59:50] WARN Could not determine content-length of response body. Set content-length of the response or set Response#chunked = true
|
10
|
+
|
data/test/mixed_rails40_test.rb
CHANGED
@@ -3,7 +3,9 @@ require 'test/unit'
|
|
3
3
|
class MixedRails40Test < Test::Unit::TestCase
|
4
4
|
|
5
5
|
def setup
|
6
|
-
|
6
|
+
bin = File.join(File.dirname(__FILE__), '..', 'bin')
|
7
|
+
examples = File.join(File.dirname(__FILE__), 'examples')
|
8
|
+
@output = `ruby #{bin}/rawk_log -f #{examples}/mixed_rails40.log`
|
7
9
|
@exit_status = $?.to_i
|
8
10
|
end
|
9
11
|
|
@@ -15,6 +17,10 @@ class MixedRails40Test < Test::Unit::TestCase
|
|
15
17
|
assert_match(/^Request +Count +Sum +Max +Median +Avg +Min +Std$/, @output)
|
16
18
|
end
|
17
19
|
|
20
|
+
def test_has_top_lists
|
21
|
+
assert_match(/^Top /, @output)
|
22
|
+
end
|
23
|
+
|
18
24
|
def test_finds_entries
|
19
25
|
assert_match(/^ItemsController#index/, @output)
|
20
26
|
assert_match(/^ItemsController#index\s+5\s/, @output)
|
data/test/rails23_test.rb
CHANGED
@@ -3,7 +3,9 @@ require 'test/unit'
|
|
3
3
|
class Rails23Test < Test::Unit::TestCase
|
4
4
|
|
5
5
|
def setup
|
6
|
-
|
6
|
+
bin = File.join(File.dirname(__FILE__), '..', 'bin')
|
7
|
+
examples = File.join(File.dirname(__FILE__), 'examples')
|
8
|
+
@output = `ruby #{bin}/rawk_log -f #{examples}/rails23.log`
|
7
9
|
@exit_status = $?.to_i
|
8
10
|
end
|
9
11
|
|
@@ -11,6 +13,10 @@ class Rails23Test < Test::Unit::TestCase
|
|
11
13
|
assert_no_match(/ItemsController/, @output)
|
12
14
|
end
|
13
15
|
|
16
|
+
def test_has_top_lists
|
17
|
+
assert_match(/^Top /, @output)
|
18
|
+
end
|
19
|
+
|
14
20
|
def test_outputs_header
|
15
21
|
assert_match(/^Request +Count +Sum\(s\) +Max +Median +Avg +Min +Std$/, @output)
|
16
22
|
end
|
data/test/rails40_test.rb
CHANGED
@@ -3,7 +3,9 @@ require 'test/unit'
|
|
3
3
|
class Rails40Test < Test::Unit::TestCase
|
4
4
|
|
5
5
|
def setup
|
6
|
-
|
6
|
+
bin = File.join(File.dirname(__FILE__), '..', 'bin')
|
7
|
+
examples = File.join(File.dirname(__FILE__), 'examples')
|
8
|
+
@output = `ruby #{bin}/rawk_log -f #{examples}/rails40.log`
|
7
9
|
@exit_status = $?.to_i
|
8
10
|
end
|
9
11
|
|
@@ -11,6 +13,10 @@ class Rails40Test < Test::Unit::TestCase
|
|
11
13
|
assert_no_match(/PostsController/, @output)
|
12
14
|
end
|
13
15
|
|
16
|
+
def test_has_top_lists
|
17
|
+
assert_match(/^Top /, @output)
|
18
|
+
end
|
19
|
+
|
14
20
|
def test_outputs_header
|
15
21
|
assert_match(/^Request +Count +Sum +Max +Median +Avg +Min +Std$/, @output)
|
16
22
|
end
|
data/test/self_test.rb
CHANGED
@@ -3,7 +3,9 @@ require 'test/unit'
|
|
3
3
|
class SelfTest < Test::Unit::TestCase
|
4
4
|
|
5
5
|
def setup
|
6
|
-
|
6
|
+
bin = File.join(File.dirname(__FILE__), '..', 'bin')
|
7
|
+
examples = File.join(File.dirname(__FILE__), 'examples')
|
8
|
+
@output = `ruby #{bin}/rawk_log -t`
|
7
9
|
@exit_status = $?.to_i
|
8
10
|
end
|
9
11
|
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'test/unit'
|
2
|
+
|
3
|
+
class UnknownTest < Test::Unit::TestCase
|
4
|
+
|
5
|
+
def setup
|
6
|
+
bin = File.join(File.dirname(__FILE__), '..', 'bin')
|
7
|
+
examples = File.join(File.dirname(__FILE__), 'examples')
|
8
|
+
@output = `ruby #{bin}/rawk_log -f #{examples}/unknown.log`
|
9
|
+
@exit_status = $?.to_i
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_outputs_header
|
13
|
+
assert_match(/^Request +Count +Sum +Max +Median +Avg +Min +Std$/, @output)
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_no_top_lists
|
17
|
+
assert_no_match(/^Top /, @output)
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_lists_zero_entries
|
21
|
+
assert_match(/^All Requests +0/, @output)
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_exit_status
|
25
|
+
assert_equal(0, @exit_status)
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rawk_log
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0
|
4
|
+
version: 2.2.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -61,17 +61,24 @@ files:
|
|
61
61
|
- Rakefile
|
62
62
|
- bin/rawk_log
|
63
63
|
- lib/rawk_log.rb
|
64
|
+
- lib/rawk_log/command.rb
|
64
65
|
- lib/rawk_log/patch_activesupport_bufferedlogger.rb
|
65
66
|
- lib/rawk_log/patch_logger.rb
|
67
|
+
- lib/rawk_log/stat.rb
|
68
|
+
- lib/rawk_log/stat_hash.rb
|
66
69
|
- lib/rawk_log/version.rb
|
67
70
|
- rawk_log.gemspec
|
71
|
+
- test/empty_test.rb
|
72
|
+
- test/examples/empty.log
|
68
73
|
- test/examples/mixed_rails40.log
|
69
74
|
- test/examples/rails23.log
|
70
75
|
- test/examples/rails40.log
|
76
|
+
- test/examples/unknown.log
|
71
77
|
- test/mixed_rails40_test.rb
|
72
78
|
- test/rails23_test.rb
|
73
79
|
- test/rails40_test.rb
|
74
80
|
- test/self_test.rb
|
81
|
+
- test/unknown_test.rb
|
75
82
|
homepage: https://github.com/ianheggie/rawk_log
|
76
83
|
licenses:
|
77
84
|
- Beerware
|
@@ -103,10 +110,14 @@ summary: This tool gives statistics for Ruby on Rails log files. The times for e
|
|
103
110
|
time or render time can be used by specifying the correct flag. The log file is
|
104
111
|
read from standard input unless the -f flag is specified.
|
105
112
|
test_files:
|
113
|
+
- test/empty_test.rb
|
114
|
+
- test/examples/empty.log
|
106
115
|
- test/examples/mixed_rails40.log
|
107
116
|
- test/examples/rails23.log
|
108
117
|
- test/examples/rails40.log
|
118
|
+
- test/examples/unknown.log
|
109
119
|
- test/mixed_rails40_test.rb
|
110
120
|
- test/rails23_test.rb
|
111
121
|
- test/rails40_test.rb
|
112
122
|
- test/self_test.rb
|
123
|
+
- test/unknown_test.rb
|