wvanbergen-request-log-analyzer 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +20 -0
- data/README +119 -0
- data/Rakefile +43 -0
- data/TODO +20 -0
- data/bin/request-log-analyzer +95 -0
- data/bin/request-log-database +79 -0
- data/lib/bashcolorizer.rb +60 -0
- data/lib/command_line/arguments.rb +129 -0
- data/lib/command_line/exceptions.rb +37 -0
- data/lib/command_line/flag.rb +51 -0
- data/lib/rails_analyzer/log_parser.rb +83 -0
- data/lib/rails_analyzer/record_inserter.rb +161 -0
- data/lib/rails_analyzer/summarizer.rb +121 -0
- data/lib/ruby-progressbar/progressbar.en.rd +103 -0
- data/lib/ruby-progressbar/progressbar.ja.rd +100 -0
- data/lib/ruby-progressbar/progressbar.rb +236 -0
- data/output/blockers.rb +11 -0
- data/output/errors.rb +9 -0
- data/output/hourly_spread.rb +28 -0
- data/output/mean_db_time.rb +7 -0
- data/output/mean_rendering_time.rb +7 -0
- data/output/mean_time.rb +7 -0
- data/output/most_requested.rb +6 -0
- data/output/timespan.rb +9 -0
- data/output/total_db_time.rb +6 -0
- data/output/total_time.rb +6 -0
- data/output/usage.rb +14 -0
- data/test/log_fragments/fragment_1.log +59 -0
- data/test/log_fragments/fragment_2.log +5 -0
- data/test/log_parser_test.rb +85 -0
- data/test/record_inserter_test.rb +42 -0
- data/test/tasks.rake +8 -0
- metadata +95 -0
@@ -0,0 +1,236 @@
|
|
1
|
+
#
|
2
|
+
# Ruby/ProgressBar - a text progress bar library
|
3
|
+
#
|
4
|
+
# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
|
5
|
+
# All rights reserved.
|
6
|
+
# This is free software with ABSOLUTELY NO WARRANTY.
|
7
|
+
#
|
8
|
+
# You can redistribute it and/or modify it under the terms
|
9
|
+
# of Ruby's license.
|
10
|
+
#
|
11
|
+
|
12
|
+
class ProgressBar
|
13
|
+
VERSION = "0.9"
|
14
|
+
|
15
|
+
def initialize (title, total, out = STDERR)
|
16
|
+
@title = title
|
17
|
+
@total = total
|
18
|
+
@out = out
|
19
|
+
@terminal_width = 80
|
20
|
+
@bar_mark = '='
|
21
|
+
@current = 0
|
22
|
+
@previous = 0
|
23
|
+
@finished_p = false
|
24
|
+
@start_time = Time.now
|
25
|
+
@previous_time = @start_time
|
26
|
+
@title_width = 24
|
27
|
+
@format = "%-#{@title_width}s %3d%% %s %s"
|
28
|
+
@format_arguments = [:title, :percentage, :bar, :stat]
|
29
|
+
clear
|
30
|
+
show
|
31
|
+
end
|
32
|
+
attr_reader :title
|
33
|
+
attr_reader :current
|
34
|
+
attr_reader :total
|
35
|
+
attr_accessor :start_time
|
36
|
+
|
37
|
+
private
|
38
|
+
def fmt_bar
|
39
|
+
bar_width = do_percentage * @terminal_width / 100
|
40
|
+
sprintf("[%s%s]",
|
41
|
+
@bar_mark * bar_width,
|
42
|
+
" " * (@terminal_width - bar_width))
|
43
|
+
end
|
44
|
+
|
45
|
+
def fmt_percentage
|
46
|
+
do_percentage
|
47
|
+
end
|
48
|
+
|
49
|
+
def fmt_stat
|
50
|
+
if @finished_p then elapsed else eta end
|
51
|
+
end
|
52
|
+
|
53
|
+
def fmt_stat_for_file_transfer
|
54
|
+
if @finished_p then
|
55
|
+
sprintf("%s %s %s", bytes, transfer_rate, elapsed)
|
56
|
+
else
|
57
|
+
sprintf("%s %s %s", bytes, transfer_rate, eta)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def fmt_title
|
62
|
+
@title[0,(@title_width - 1)] + ":"
|
63
|
+
end
|
64
|
+
|
65
|
+
def convert_bytes (bytes)
|
66
|
+
if bytes < 1024
|
67
|
+
sprintf("%6dB", bytes)
|
68
|
+
elsif bytes < 1024 * 1000 # 1000kb
|
69
|
+
sprintf("%5.1fKB", bytes.to_f / 1024)
|
70
|
+
elsif bytes < 1024 * 1024 * 1000 # 1000mb
|
71
|
+
sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
|
72
|
+
else
|
73
|
+
sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def transfer_rate
|
78
|
+
bytes_per_second = @current.to_f / (Time.now - @start_time)
|
79
|
+
sprintf("%s/s", convert_bytes(bytes_per_second))
|
80
|
+
end
|
81
|
+
|
82
|
+
def bytes
|
83
|
+
convert_bytes(@current)
|
84
|
+
end
|
85
|
+
|
86
|
+
def format_time (t)
|
87
|
+
t = t.to_i
|
88
|
+
sec = t % 60
|
89
|
+
min = (t / 60) % 60
|
90
|
+
hour = t / 3600
|
91
|
+
sprintf("%02d:%02d:%02d", hour, min, sec);
|
92
|
+
end
|
93
|
+
|
94
|
+
# ETA stands for Estimated Time of Arrival.
|
95
|
+
def eta
|
96
|
+
if @current == 0
|
97
|
+
white("ETA: --:--:--")
|
98
|
+
else
|
99
|
+
elapsed = Time.now - @start_time
|
100
|
+
eta = elapsed * @total / @current - elapsed;
|
101
|
+
sprintf(white("ETA: %s"), format_time(eta))
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def elapsed
|
106
|
+
elapsed = Time.now - @start_time
|
107
|
+
sprintf("Time: %s", format_time(elapsed))
|
108
|
+
end
|
109
|
+
|
110
|
+
def eol
|
111
|
+
if @finished_p then "\n" else "\r" end
|
112
|
+
end
|
113
|
+
|
114
|
+
def do_percentage
|
115
|
+
if @total.zero?
|
116
|
+
100
|
117
|
+
else
|
118
|
+
@current * 100 / @total
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
def get_width
|
123
|
+
# FIXME: I don't know how portable it is.
|
124
|
+
default_width = 80
|
125
|
+
begin
|
126
|
+
tiocgwinsz = 0x5413
|
127
|
+
data = [0, 0, 0, 0].pack("SSSS")
|
128
|
+
if @out.ioctl(tiocgwinsz, data) >= 0 then
|
129
|
+
rows, cols, xpixels, ypixels = data.unpack("SSSS")
|
130
|
+
if cols >= 0 then cols else default_width end
|
131
|
+
else
|
132
|
+
default_width
|
133
|
+
end
|
134
|
+
rescue Exception
|
135
|
+
default_width
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
def show
|
140
|
+
arguments = @format_arguments.map {|method|
|
141
|
+
method = sprintf("fmt_%s", method)
|
142
|
+
send(method)
|
143
|
+
}
|
144
|
+
line = sprintf(@format, *arguments)
|
145
|
+
|
146
|
+
width = get_width
|
147
|
+
if line.length == width - 1
|
148
|
+
@out.print(line + eol)
|
149
|
+
@out.flush
|
150
|
+
elsif line.length >= width
|
151
|
+
@terminal_width = [@terminal_width - (line.length - width + 1), 0].max
|
152
|
+
if @terminal_width == 0 then @out.print(line + eol) else show end
|
153
|
+
else # line.length < width - 1
|
154
|
+
@terminal_width += width - line.length + 1
|
155
|
+
show
|
156
|
+
end
|
157
|
+
@previous_time = Time.now
|
158
|
+
end
|
159
|
+
|
160
|
+
def show_if_needed
|
161
|
+
if @total.zero?
|
162
|
+
cur_percentage = 100
|
163
|
+
prev_percentage = 0
|
164
|
+
else
|
165
|
+
cur_percentage = (@current * 100 / @total).to_i
|
166
|
+
prev_percentage = (@previous * 100 / @total).to_i
|
167
|
+
end
|
168
|
+
|
169
|
+
# Use "!=" instead of ">" to support negative changes
|
170
|
+
if cur_percentage != prev_percentage ||
|
171
|
+
Time.now - @previous_time >= 1 || @finished_p
|
172
|
+
show
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
public
|
177
|
+
def clear
|
178
|
+
@out.print "\r"
|
179
|
+
@out.print(" " * (get_width - 1))
|
180
|
+
@out.print "\r"
|
181
|
+
end
|
182
|
+
|
183
|
+
def finish
|
184
|
+
@current = @total
|
185
|
+
@finished_p = true
|
186
|
+
show
|
187
|
+
end
|
188
|
+
|
189
|
+
def finished?
|
190
|
+
@finished_p
|
191
|
+
end
|
192
|
+
|
193
|
+
def file_transfer_mode
|
194
|
+
@format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
|
195
|
+
end
|
196
|
+
|
197
|
+
def format= (format)
|
198
|
+
@format = format
|
199
|
+
end
|
200
|
+
|
201
|
+
def format_arguments= (arguments)
|
202
|
+
@format_arguments = arguments
|
203
|
+
end
|
204
|
+
|
205
|
+
def halt
|
206
|
+
@finished_p = true
|
207
|
+
show
|
208
|
+
end
|
209
|
+
|
210
|
+
def inc (step = 1)
|
211
|
+
@current += step
|
212
|
+
@current = @total if @current > @total
|
213
|
+
show_if_needed
|
214
|
+
@previous = @current
|
215
|
+
end
|
216
|
+
|
217
|
+
def set (count)
|
218
|
+
if count < 0 || count > @total
|
219
|
+
raise "invalid count: #{count} (total: #{@total})"
|
220
|
+
end
|
221
|
+
@current = count
|
222
|
+
show_if_needed
|
223
|
+
@previous = @current
|
224
|
+
end
|
225
|
+
|
226
|
+
def inspect
|
227
|
+
"#<ProgressBar:#{@current}/#{@total}>"
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
class ReversedProgressBar < ProgressBar
|
232
|
+
def do_percentage
|
233
|
+
100 - super
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
data/output/blockers.rb
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
# Print requests that took more than a second to complete, sorted by their frequency.
|
2
|
+
amount = $arguments[:amount] || 10
|
3
|
+
|
4
|
+
puts
|
5
|
+
puts "Mongrel process blockers (> #{$summarizer.blocker_duration} seconds)"
|
6
|
+
puts green("========================================================================")
|
7
|
+
|
8
|
+
$summarizer.sort_blockers_by(:count).reverse[0, amount.to_i].each do |a|
|
9
|
+
puts "#{a[0].ljust(50)}: %10.03fs [#{green("%d requests")}]" % [a[1][:total_time], a[1][:count]]
|
10
|
+
end
|
11
|
+
|
data/output/errors.rb
ADDED
@@ -0,0 +1,9 @@
|
|
1
|
+
# Print errors that occured often
|
2
|
+
amount = $arguments[:amount] || 10
|
3
|
+
puts
|
4
|
+
puts "Errors"
|
5
|
+
puts green("========================================================================")
|
6
|
+
$summarizer.sort_errors_by(:count).reverse[0, amount.to_i].each do |a|
|
7
|
+
puts "#{(a[0] + 'Error').ljust(50)}: [#{green("%d requests")}]" % a[1][:count]
|
8
|
+
puts blue(' -> ' + (a[1][:exception_strings].invert[ a[1][:exception_strings].values.max ])[0..79])
|
9
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# Draws a graph containing the average amound of requests per hour per day
|
2
|
+
if $summarizer.request_time_graph?
|
3
|
+
|
4
|
+
max_request_graph = $summarizer.request_time_graph.max / $summarizer.duration
|
5
|
+
deviation = max_request_graph / 20
|
6
|
+
deviation = 1 if deviation == 0
|
7
|
+
color_cutoff = 15
|
8
|
+
|
9
|
+
puts
|
10
|
+
puts "Requests graph - per hour"
|
11
|
+
puts green("========================================================================")
|
12
|
+
|
13
|
+
(0..23).each do |a|
|
14
|
+
requests = $summarizer.request_time_graph[a] / $summarizer.duration
|
15
|
+
display_chars = requests / deviation
|
16
|
+
|
17
|
+
if display_chars >= color_cutoff
|
18
|
+
display_chars_string = green(' ' * color_cutoff) + red(' ' * (display_chars - color_cutoff))
|
19
|
+
else
|
20
|
+
display_chars_string = green(' ' * display_chars)
|
21
|
+
end
|
22
|
+
|
23
|
+
puts "#{a.to_s.rjust(10)}:00 - #{('[' + requests.to_s + ' req.]').ljust(15)} : #{display_chars_string}"
|
24
|
+
end
|
25
|
+
else
|
26
|
+
puts
|
27
|
+
puts "Hourly spread not available"
|
28
|
+
end
|
@@ -0,0 +1,7 @@
|
|
1
|
+
# Prints a table sorted by the highest database times.
|
2
|
+
amount = $arguments[:amount] || 10
|
3
|
+
puts
|
4
|
+
puts "Top #{amount} worst DB offenders - mean time"
|
5
|
+
puts green("========================================================================")
|
6
|
+
print_table($summarizer, :mean_db_time, amount)
|
7
|
+
|
@@ -0,0 +1,7 @@
|
|
1
|
+
# Prints a table showing the slowest renderes
|
2
|
+
amount = $arguments[:amount] || 10
|
3
|
+
puts
|
4
|
+
puts "Top #{amount} slow renderers - mean time"
|
5
|
+
puts green("========================================================================")
|
6
|
+
print_table($summarizer, :mean_rendering_time, amount)
|
7
|
+
|
data/output/mean_time.rb
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
# Prints table sorted by the duration of the requests
|
2
|
+
amount = $arguments[:amount] || 10
|
3
|
+
puts
|
4
|
+
puts "Top #{amount} actions by time - per request mean"
|
5
|
+
puts green("========================================================================")
|
6
|
+
|
7
|
+
print_table($summarizer, :mean_time, amount)
|
@@ -0,0 +1,6 @@
|
|
1
|
+
# Prints a table sorted by the most frequently requested actions
|
2
|
+
amount = $arguments[:amount] || 10
|
3
|
+
puts
|
4
|
+
puts "Top #{amount} most requested actions"
|
5
|
+
puts green("========================================================================")
|
6
|
+
print_table($summarizer, :count, amount)
|
data/output/timespan.rb
ADDED
@@ -0,0 +1,9 @@
|
|
1
|
+
# Prints the total timespan found in the parsed log files.
|
2
|
+
puts
|
3
|
+
puts green("========================================================================")
|
4
|
+
|
5
|
+
if $summarizer.has_timestamps?
|
6
|
+
puts "Timestamp first request: #{$summarizer.first_request_at}"
|
7
|
+
puts "Timestamp last request: #{$summarizer.last_request_at}"
|
8
|
+
puts "Total time analyzed: #{$summarizer.duration} days"
|
9
|
+
end
|
@@ -0,0 +1,6 @@
|
|
1
|
+
# Prints a list of the actions that spend most of their time waiting for database results.
|
2
|
+
amount = $arguments[:amount] || 10
|
3
|
+
puts
|
4
|
+
puts "Top #{amount} worst DB offenders - cumulative time"
|
5
|
+
puts green("========================================================================")
|
6
|
+
print_table($summarizer, :total_db_time, amount)
|
@@ -0,0 +1,6 @@
|
|
1
|
+
# Prints a list ordered by the requests that took the most time in total.
|
2
|
+
amount = $arguments[:amount] || 10
|
3
|
+
puts
|
4
|
+
puts "Top #{amount} actions by time - cumulative"
|
5
|
+
puts green("========================================================================")
|
6
|
+
print_table($summarizer, :total_time, amount)
|
data/output/usage.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
# Prints usage table
|
2
|
+
puts "Usage: request-log-analyzer [LOGFILES*] <OPTIONS>"
|
3
|
+
puts
|
4
|
+
puts "Options:"
|
5
|
+
puts " --fast, -t: Only use completed requests"
|
6
|
+
puts " --guess-database-time, -g: Guesses the database duration of requests"
|
7
|
+
puts " --output, -o: Comma-separated list of reports to show"
|
8
|
+
puts " --amount, -c: Displays the top <amount> elements in the reports"
|
9
|
+
puts " --colorize, -z: Fancy bash coloring"
|
10
|
+
puts
|
11
|
+
puts "Examples:"
|
12
|
+
puts " request-log-analyzer development.log"
|
13
|
+
puts " request-log-analyzer mongrel.0.log mongrel.1.log mongrel.2.log -g -f -o mean_time,most_requested,blockers -c 20 -z"
|
14
|
+
puts
|
@@ -0,0 +1,59 @@
|
|
1
|
+
Processing DashboardController#index (for 130.89.162.199 at 2008-08-14 21:16:25) [GET]
|
2
|
+
Session ID: BAh7CToMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUvMjM1MCIKZmxhc2hJ
|
3
|
+
QzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNoSGFzaHsABjoKQHVz
|
4
|
+
ZWR7ADoNbGFuZ3VhZ2VvOhNMb2NhbGU6Ok9iamVjdBI6CUB3aW4wOg1AY291
|
5
|
+
bnRyeSIHTkw6CkBoYXNoaf3L2Js6DkBvcmlnX3N0ciIKbmwtTkw6DUBpc28z
|
6
|
+
MDY2MDoNQGNoYXJzZXQiClVURi04Og5AbGFuZ3VhZ2UiB25sOg5AbW9kaWZp
|
7
|
+
ZXIwOgtAcG9zaXgiCm5sX05MOg1AZ2VuZXJhbCIKbmxfTkw6DUB2YXJpYW50
|
8
|
+
MDoOQGZhbGxiYWNrMDoMQHNjcmlwdDA6DnBlcnNvbl9pZGkCMgc=--7918aed37151c13360cd370c37b541f136146fbd
|
9
|
+
Parameters: {"action"=>"index", "controller"=>"dashboard"}
|
10
|
+
Set language to: nl_NL
|
11
|
+
Rendering template within layouts/priscilla
|
12
|
+
Rendering dashboard/index
|
13
|
+
Completed in 0.22699 (4 reqs/sec) | Rendering: 0.02667 (11%) | DB: 0.03057 (13%) | 200 OK [https://www.example.com/]
|
14
|
+
|
15
|
+
|
16
|
+
Processing PeopleController#index (for 130.89.162.199 at 2008-08-14 21:16:30) [GET]
|
17
|
+
Session ID: BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo
|
18
|
+
SGFzaHsABjoKQHVzZWR7ADoMcmVmZXJlciIQL3ByaXNjaWxsYS86DnBlcnNv
|
19
|
+
bl9pZGkCMgc6DWxhbmd1YWdlbzoTTG9jYWxlOjpPYmplY3QSOg1AY291bnRy
|
20
|
+
eSIHTkw6CUB3aW4wOg5Ab3JpZ19zdHIiCm5sLU5MOgpAaGFzaGn9y9ibOg5A
|
21
|
+
bGFuZ3VhZ2UiB25sOg1AY2hhcnNldCIKVVRGLTg6DUBpc28zMDY2MDoOQG1v
|
22
|
+
ZGlmaWVyMDoLQHBvc2l4IgpubF9OTDoNQHZhcmlhbnQwOg1AZ2VuZXJhbCIK
|
23
|
+
bmxfTkw6DEBzY3JpcHQwOg5AZmFsbGJhY2sw--48cbe3788ef27f6005f8e999610a42af6e90ffb3
|
24
|
+
Parameters: {"commit"=>"Zoek", "action"=>"index", "q"=>"gaby", "controller"=>"people"}
|
25
|
+
Set language to: nl_NL
|
26
|
+
Redirected to https://www.example.com/people/2545
|
27
|
+
Completed in 0.04759 (21 reqs/sec) | DB: 0.03719 (78%) | 302 Found [https://www.example.com/people?q=gaby&commit=Zoek]
|
28
|
+
|
29
|
+
|
30
|
+
Processing PeopleController#show (for 130.89.162.199 at 2008-08-14 21:16:30) [GET]
|
31
|
+
Session ID: BAh7CToMcmVmZXJlciIpL3ByaXNjaWxsYS9wZW9wbGU/cT1nYWJ5JmNvbW1p
|
32
|
+
dD1ab2VrIgpmbGFzaElDOidBY3Rpb25Db250cm9sbGVyOjpGbGFzaDo6Rmxh
|
33
|
+
c2hIYXNoewAGOgpAdXNlZHsAOg1sYW5ndWFnZW86E0xvY2FsZTo6T2JqZWN0
|
34
|
+
EjoJQHdpbjA6DUBjb3VudHJ5IgdOTDoKQGhhc2hp/cvYmzoOQG9yaWdfc3Ry
|
35
|
+
IgpubC1OTDoNQGlzbzMwNjYwOg1AY2hhcnNldCIKVVRGLTg6DkBsYW5ndWFn
|
36
|
+
ZSIHbmw6DkBtb2RpZmllcjA6C0Bwb3NpeCIKbmxfTkw6DUBnZW5lcmFsIgpu
|
37
|
+
bF9OTDoNQHZhcmlhbnQwOg5AZmFsbGJhY2swOgxAc2NyaXB0MDoOcGVyc29u
|
38
|
+
X2lkaQIyBw==--3ad1948559448522a49d289a2a89dc7ccbe8847a
|
39
|
+
Parameters: {"action"=>"show", "id"=>"2545", "controller"=>"people"}
|
40
|
+
Set language to: nl_NL
|
41
|
+
Rendering template within layouts/priscilla
|
42
|
+
Rendering people/show
|
43
|
+
person: Gaby Somers, study_year: 2008/2009
|
44
|
+
Completed in 0.29077 (3 reqs/sec) | Rendering: 0.24187 (83%) | DB: 0.04030 (13%) | 200 OK [https://www.example.com/people/2545]
|
45
|
+
|
46
|
+
|
47
|
+
Processing PeopleController#picture (for 130.89.162.199 at 2008-08-14 21:16:35) [GET]
|
48
|
+
Session ID: BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo
|
49
|
+
SGFzaHsABjoKQHVzZWR7ADoMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUv
|
50
|
+
MjU0NToOcGVyc29uX2lkaQIyBzoNbGFuZ3VhZ2VvOhNMb2NhbGU6Ok9iamVj
|
51
|
+
dBI6DUBjb3VudHJ5IgdOTDoJQHdpbjA6DkBvcmlnX3N0ciIKbmwtTkw6CkBo
|
52
|
+
YXNoaf3L2Js6DkBsYW5ndWFnZSIHbmw6DUBjaGFyc2V0IgpVVEYtODoNQGlz
|
53
|
+
bzMwNjYwOg5AbW9kaWZpZXIwOgtAcG9zaXgiCm5sX05MOg1AdmFyaWFudDA6
|
54
|
+
DUBnZW5lcmFsIgpubF9OTDoMQHNjcmlwdDA6DkBmYWxsYmFjazA=--797a33f280a482647111397d138d0918f2658167
|
55
|
+
Parameters: {"action"=>"picture", "id"=>"2545", "controller"=>"people"}
|
56
|
+
Set language to: nl_NL
|
57
|
+
Rendering template within layouts/priscilla
|
58
|
+
Rendering people/picture
|
59
|
+
Completed in 0.05383 (18 reqs/sec) | Rendering: 0.04622 (85%) | DB: 0.00206 (3%) | 200 OK [https://www.example.com/people/2545/picture]
|
@@ -0,0 +1,5 @@
|
|
1
|
+
Jul 13 06:25:58 10.1.1.32 app_p [1957]: Processing EmployeeController#index (for 10.1.1.33 at 2008-07-13 06:25:58) [GET]
|
2
|
+
Jul 13 06:25:58 10.1.1.32 app_p [1957]: Session ID: bd1810833653be11c38ad1e5675635bd
|
3
|
+
Jul 13 06:25:58 10.1.1.32 app_p [1957]: Parameters: {"format"=>"xml", "action"=>"index}
|
4
|
+
Jul 13 06:25:58 10.1.1.32 app_p [1957]: Rendering employees
|
5
|
+
Jul 13 06:25:58 10.1.1.32 app_p [1957]: Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://example.com/employee.xml]
|
@@ -0,0 +1,85 @@
|
|
1
|
+
require 'test/unit'
|
2
|
+
|
3
|
+
require "#{File.dirname(__FILE__)}/../lib/rails_analyzer/log_parser"
|
4
|
+
|
5
|
+
class LogParserTest < Test::Unit::TestCase
|
6
|
+
|
7
|
+
def fragment_file(number)
|
8
|
+
"#{File.dirname(__FILE__)}/log_fragments/fragment_#{number}.log"
|
9
|
+
end
|
10
|
+
|
11
|
+
|
12
|
+
def test_progress_messages
|
13
|
+
log_file = fragment_file(1)
|
14
|
+
|
15
|
+
finished_encountered = false
|
16
|
+
file_size = File.size(log_file)
|
17
|
+
|
18
|
+
previous_pos = -1
|
19
|
+
parser = RailsAnalyzer::LogParser.new(log_file)
|
20
|
+
parser.progress do |pos, total|
|
21
|
+
assert_equal file_size, total
|
22
|
+
if pos == :finished
|
23
|
+
finished_encountered = true
|
24
|
+
else
|
25
|
+
assert pos <= total
|
26
|
+
assert pos > previous_pos
|
27
|
+
previous_pos = pos
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
# now parse the file
|
32
|
+
parser.each(:started) { }
|
33
|
+
|
34
|
+
assert finished_encountered, "A finished event should have been fired"
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_parse_mongrel_log_fragment
|
38
|
+
count = 0
|
39
|
+
parser = RailsAnalyzer::LogParser.new(fragment_file(1)).each(:started) { count += 1 }
|
40
|
+
assert_equal 4, count
|
41
|
+
|
42
|
+
count = 0
|
43
|
+
parser = RailsAnalyzer::LogParser.new(fragment_file(1)).each(:completed) { count += 1 }
|
44
|
+
assert_equal 4, count
|
45
|
+
|
46
|
+
count = 0
|
47
|
+
parser = RailsAnalyzer::LogParser.new(fragment_file(1)).each(:started, :completed) { count += 1 }
|
48
|
+
assert_equal 8, count
|
49
|
+
end
|
50
|
+
|
51
|
+
def test_parse_syslog_fragment
|
52
|
+
count = 0
|
53
|
+
parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:started) { count += 1 }
|
54
|
+
assert_equal 1, count
|
55
|
+
|
56
|
+
count = 0
|
57
|
+
parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:completed) { count += 1 }
|
58
|
+
assert_equal 1, count
|
59
|
+
|
60
|
+
count = 0
|
61
|
+
parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:started, :completed) { count += 1 }
|
62
|
+
assert_equal 2, count
|
63
|
+
end
|
64
|
+
|
65
|
+
def test_parse_syslog_fragment_content
|
66
|
+
# this test only works because there is only one requests in the fragment
|
67
|
+
parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:started) do |request|
|
68
|
+
assert_equal "EmployeeController", request[:controller]
|
69
|
+
assert_equal "index", request[:action]
|
70
|
+
assert_equal "GET", request[:method]
|
71
|
+
assert_equal '10.1.1.33', request[:ip]
|
72
|
+
assert_equal '2008-07-13 06:25:58', request[:timestamp]
|
73
|
+
end
|
74
|
+
|
75
|
+
parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:completed) do |request|
|
76
|
+
assert_equal "http://example.com/employee.xml", request[:url]
|
77
|
+
assert_equal 200, request[:status]
|
78
|
+
assert_equal 0.21665, request[:duration]
|
79
|
+
assert_equal 0.00926, request[:rendering]
|
80
|
+
assert_equal 0.0, request[:db]
|
81
|
+
end
|
82
|
+
|
83
|
+
end
|
84
|
+
|
85
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'test/unit'
|
2
|
+
|
3
|
+
require "#{File.dirname(__FILE__)}/../lib/rails_analyzer/log_parser"
|
4
|
+
require "#{File.dirname(__FILE__)}/../lib/rails_analyzer/record_inserter"
|
5
|
+
|
6
|
+
class RecordInserterTest < Test::Unit::TestCase
|
7
|
+
|
8
|
+
def fragment_file(number)
|
9
|
+
"#{File.dirname(__FILE__)}/log_fragments/fragment_#{number}.log"
|
10
|
+
end
|
11
|
+
|
12
|
+
def setup
|
13
|
+
File.delete('_tmp.db') if File.exist?('_tmp.db')
|
14
|
+
end
|
15
|
+
|
16
|
+
def teardown
|
17
|
+
File.delete('_tmp.db') if File.exist?('_tmp.db')
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_insert_log_fragment
|
21
|
+
|
22
|
+
db = RailsAnalyzer::RecordInserter.insert_batch_into('_tmp.db') do |batch|
|
23
|
+
RailsAnalyzer::LogParser.new(fragment_file(1)).each { |request| batch.insert(request) }
|
24
|
+
end
|
25
|
+
|
26
|
+
assert_equal 4, db.database.get_first_value("SELECT COUNT(*) FROM started_requests").to_i
|
27
|
+
assert_equal 4, db.database.get_first_value("SELECT COUNT(*) FROM completed_requests").to_i
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_insert_multiple_fragments
|
31
|
+
RailsAnalyzer::RecordInserter.insert_batch_into('_tmp.db') do |batch|
|
32
|
+
RailsAnalyzer::LogParser.new(fragment_file(1)).each { |request| batch.insert(request) }
|
33
|
+
end
|
34
|
+
|
35
|
+
db = RailsAnalyzer::RecordInserter.insert_batch_into('_tmp.db') do |batch|
|
36
|
+
RailsAnalyzer::LogParser.new(fragment_file(2)).each { |request| batch.insert(request) }
|
37
|
+
end
|
38
|
+
assert_equal 5, db.database.get_first_value("SELECT COUNT(*) FROM started_requests").to_i
|
39
|
+
assert_equal 5, db.database.get_first_value("SELECT COUNT(*) FROM completed_requests").to_i
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
data/test/tasks.rake
ADDED
metadata
ADDED
@@ -0,0 +1,95 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: wvanbergen-request-log-analyzer
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Willem van Bergen
|
8
|
+
- Bart ten Brinke
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
|
13
|
+
date: 2008-08-29 00:00:00 -07:00
|
14
|
+
default_executable: request-log-analyzer
|
15
|
+
dependencies: []
|
16
|
+
|
17
|
+
description: Rails log analyzer's purpose is to find what actions are best candidates for optimization. This tool will parse all requests in the Rails logfile and aggregate the information. Once it is finished parsing the log file, it will show the requests that take op most server time using various metrics.
|
18
|
+
email: willem@vanbergen.org
|
19
|
+
executables:
|
20
|
+
- request-log-analyzer
|
21
|
+
- request-log-database
|
22
|
+
extensions: []
|
23
|
+
|
24
|
+
extra_rdoc_files: []
|
25
|
+
|
26
|
+
files:
|
27
|
+
- LICENSE
|
28
|
+
- README
|
29
|
+
- Rakefile
|
30
|
+
- TODO
|
31
|
+
- bin
|
32
|
+
- bin/request-log-analyzer
|
33
|
+
- bin/request-log-database
|
34
|
+
- lib
|
35
|
+
- lib/bashcolorizer.rb
|
36
|
+
- lib/command_line
|
37
|
+
- lib/command_line/arguments.rb
|
38
|
+
- lib/command_line/exceptions.rb
|
39
|
+
- lib/command_line/flag.rb
|
40
|
+
- lib/rails_analyzer
|
41
|
+
- lib/rails_analyzer/log_parser.rb
|
42
|
+
- lib/rails_analyzer/record_inserter.rb
|
43
|
+
- lib/rails_analyzer/summarizer.rb
|
44
|
+
- lib/ruby-progressbar
|
45
|
+
- lib/ruby-progressbar/progressbar.en.rd
|
46
|
+
- lib/ruby-progressbar/progressbar.ja.rd
|
47
|
+
- lib/ruby-progressbar/progressbar.rb
|
48
|
+
- output
|
49
|
+
- output/blockers.rb
|
50
|
+
- output/errors.rb
|
51
|
+
- output/hourly_spread.rb
|
52
|
+
- output/mean_db_time.rb
|
53
|
+
- output/mean_rendering_time.rb
|
54
|
+
- output/mean_time.rb
|
55
|
+
- output/most_requested.rb
|
56
|
+
- output/timespan.rb
|
57
|
+
- output/total_db_time.rb
|
58
|
+
- output/total_time.rb
|
59
|
+
- output/usage.rb
|
60
|
+
- test
|
61
|
+
- test/log_fragments
|
62
|
+
- test/log_fragments/fragment_1.log
|
63
|
+
- test/log_fragments/fragment_2.log
|
64
|
+
- test/log_parser_test.rb
|
65
|
+
- test/record_inserter_test.rb
|
66
|
+
- test/tasks.rake
|
67
|
+
has_rdoc: false
|
68
|
+
homepage: http://github.com/wvanbergen/request-log-analyzer/wikis
|
69
|
+
post_install_message:
|
70
|
+
rdoc_options: []
|
71
|
+
|
72
|
+
require_paths:
|
73
|
+
- lib
|
74
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
75
|
+
requirements:
|
76
|
+
- - ">="
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
version: "0"
|
79
|
+
version:
|
80
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
81
|
+
requirements:
|
82
|
+
- - ">="
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
version: "0"
|
85
|
+
version:
|
86
|
+
requirements: []
|
87
|
+
|
88
|
+
rubyforge_project:
|
89
|
+
rubygems_version: 1.2.0
|
90
|
+
signing_key:
|
91
|
+
specification_version: 2
|
92
|
+
summary: A command line tool to analyze Rails logs
|
93
|
+
test_files:
|
94
|
+
- test/log_parser_test.rb
|
95
|
+
- test/record_inserter_test.rb
|