request-log-analyzer 1.1.0 → 1.1.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +4 -3
- data/bin/request-log-analyzer +4 -5
- data/lib/cli/command_line_arguments.rb +2 -2
- data/lib/request_log_analyzer.rb +28 -6
- data/lib/request_log_analyzer/{aggregator/base.rb → aggregator.rb} +5 -1
- data/lib/request_log_analyzer/aggregator/summarizer.rb +2 -3
- data/lib/request_log_analyzer/controller.rb +11 -16
- data/lib/request_log_analyzer/file_format.rb +71 -38
- data/lib/request_log_analyzer/file_format/merb.rb +32 -26
- data/lib/request_log_analyzer/file_format/rails.rb +73 -71
- data/lib/request_log_analyzer/file_format/rails_development.rb +93 -95
- data/lib/request_log_analyzer/filter.rb +38 -0
- data/lib/request_log_analyzer/filter/anonimize.rb +1 -1
- data/lib/request_log_analyzer/line_definition.rb +1 -1
- data/lib/request_log_analyzer/output.rb +6 -8
- data/lib/request_log_analyzer/output/fixed_width.rb +133 -117
- data/lib/request_log_analyzer/output/html.rb +138 -60
- data/lib/request_log_analyzer/request.rb +3 -1
- data/lib/request_log_analyzer/{source/base.rb → source.rb} +5 -0
- data/lib/request_log_analyzer/source/{log_file.rb → log_parser.rb} +15 -6
- data/lib/request_log_analyzer/tracker.rb +58 -0
- data/lib/request_log_analyzer/tracker/category.rb +7 -8
- data/lib/request_log_analyzer/tracker/duration.rb +15 -12
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -8
- data/lib/request_log_analyzer/tracker/timespan.rb +10 -10
- data/spec/controller_spec.rb +5 -4
- data/spec/database_inserter_spec.rb +5 -8
- data/spec/file_format_spec.rb +2 -2
- data/spec/file_formats/spec_format.rb +2 -1
- data/spec/filter_spec.rb +0 -3
- data/spec/log_parser_spec.rb +6 -6
- data/spec/merb_format_spec.rb +38 -38
- data/spec/rails_format_spec.rb +2 -2
- data/spec/request_spec.rb +2 -2
- data/spec/spec_helper.rb +3 -37
- data/tasks/github-gem.rake +2 -1
- metadata +7 -8
- data/lib/request_log_analyzer/filter/base.rb +0 -32
- data/lib/request_log_analyzer/log_parser.rb +0 -173
- data/lib/request_log_analyzer/tracker/base.rb +0 -54
@@ -1,80 +1,158 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
1
|
+
module RequestLogAnalyzer::Output
|
2
|
+
|
3
|
+
class HTML < Base
|
4
|
+
|
5
|
+
# def initialize(io, options = {})
|
6
|
+
# super(io, options)
|
7
|
+
# end
|
8
|
+
|
9
|
+
def print(str)
|
10
|
+
@io << str
|
11
|
+
end
|
10
12
|
|
11
|
-
|
13
|
+
alias :<< :print
|
12
14
|
|
13
|
-
|
14
|
-
|
15
|
-
|
15
|
+
def puts(str = '')
|
16
|
+
@io << str << "<br/>\n"
|
17
|
+
end
|
16
18
|
|
17
|
-
|
18
|
-
|
19
|
-
|
19
|
+
def title(title)
|
20
|
+
@io.puts(tag(:h2, title))
|
21
|
+
end
|
20
22
|
|
21
|
-
|
22
|
-
|
23
|
-
|
23
|
+
def line(*font)
|
24
|
+
@io.puts(tag(:hr))
|
25
|
+
end
|
24
26
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
27
|
+
def link(text, url = nil)
|
28
|
+
url = text if url.nil?
|
29
|
+
tag(:a, text, :href => url)
|
30
|
+
end
|
29
31
|
|
30
|
-
|
31
|
-
|
32
|
-
|
32
|
+
def table(*columns, &block)
|
33
|
+
rows = Array.new
|
34
|
+
yield(rows)
|
33
35
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
36
|
+
@io << tag(:table, {:id => 'mytable', :cellspacing => 0}) do |content|
|
37
|
+
if table_has_header?(columns)
|
38
|
+
content << tag(:tr) do
|
39
|
+
columns.map { |col| tag(:th, col[:title]) }.join("\n")
|
40
|
+
end
|
38
41
|
end
|
39
|
-
end
|
40
42
|
|
41
|
-
|
42
|
-
|
43
|
-
|
43
|
+
odd = false
|
44
|
+
rows.each do |row|
|
45
|
+
odd = !odd
|
46
|
+
content << tag(:tr) do
|
47
|
+
if odd
|
48
|
+
row.map { |cell| tag(:td, cell, :class => 'alt') }.join("\n")
|
49
|
+
else
|
50
|
+
row.map { |cell| tag(:td, cell) }.join("\n")
|
51
|
+
end
|
52
|
+
end
|
44
53
|
end
|
45
54
|
end
|
46
|
-
end
|
47
55
|
|
48
|
-
|
56
|
+
end
|
49
57
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
58
|
+
def header
|
59
|
+
@io << "<html>"
|
60
|
+
@io << tag(:head) do |headers|
|
61
|
+
headers << tag(:title, 'Request-log-analyzer report')
|
62
|
+
headers << tag(:style, '
|
63
|
+
body {
|
64
|
+
font: normal 11px auto "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
65
|
+
color: #4f6b72;
|
66
|
+
background: #E6EAE9;
|
67
|
+
padding-left:20px;
|
68
|
+
padding-top:20px;
|
69
|
+
padding-bottom:20px;
|
70
|
+
}
|
71
|
+
|
72
|
+
a {
|
73
|
+
color: #c75f3e;
|
74
|
+
}
|
75
|
+
|
76
|
+
.color_bar {
|
77
|
+
border: 1px solid;
|
78
|
+
height:10px;
|
79
|
+
background: #CAE8EA;
|
80
|
+
}
|
81
|
+
|
82
|
+
#mytable {
|
83
|
+
width: 700px;
|
84
|
+
padding: 0;
|
85
|
+
margin: 0;
|
86
|
+
padding-bottom:10px;
|
87
|
+
}
|
88
|
+
|
89
|
+
caption {
|
90
|
+
padding: 0 0 5px 0;
|
91
|
+
width: 700px;
|
92
|
+
font: italic 11px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
93
|
+
text-align: right;
|
94
|
+
}
|
95
|
+
|
96
|
+
th {
|
97
|
+
font: bold 11px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
98
|
+
color: #4f6b72;
|
99
|
+
border-right: 1px solid #C1DAD7;
|
100
|
+
border-bottom: 1px solid #C1DAD7;
|
101
|
+
border-top: 1px solid #C1DAD7;
|
102
|
+
letter-spacing: 2px;
|
103
|
+
text-transform: uppercase;
|
104
|
+
text-align: left;
|
105
|
+
padding: 6px 6px 6px 12px;
|
106
|
+
background: #CAE8EA url(images/bg_header.jpg) no-repeat;
|
107
|
+
}
|
108
|
+
|
109
|
+
td {
|
110
|
+
font: bold 11px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
111
|
+
border-right: 1px solid #C1DAD7;
|
112
|
+
border-bottom: 1px solid #C1DAD7;
|
113
|
+
background: #fff;
|
114
|
+
padding: 6px 6px 6px 12px;
|
115
|
+
color: #4f6b72;
|
116
|
+
}
|
117
|
+
|
118
|
+
td.alt {
|
119
|
+
background: #F5FAFA;
|
120
|
+
color: #797268;
|
121
|
+
}
|
122
|
+
', :type => "text/css")
|
123
|
+
end
|
124
|
+
@io << '<body>'
|
125
|
+
@io << tag(:h1, 'Request-log-analyzer summary report')
|
126
|
+
@io << tag(:p, "Version #{RequestLogAnalyzer::VERSION} - written by Willem van Bergen and Bart ten Brinke")
|
54
127
|
end
|
55
|
-
@io << '<body>'
|
56
|
-
end
|
57
128
|
|
58
|
-
|
59
|
-
|
60
|
-
|
129
|
+
def footer
|
130
|
+
@io << tag(:hr) << tag(:h2, 'Thanks for using request-log-analyzer')
|
131
|
+
@io << tag(:p, 'Please visit the ' + link('Request-log-analyzer website', 'http://github.com/wvanbergen/request-log-analyzer'))
|
132
|
+
@io << "</body></html>\n"
|
133
|
+
end
|
61
134
|
|
62
|
-
|
135
|
+
protected
|
63
136
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
"<#{tag}#{attributes}>#{content}</#{tag}>"
|
71
|
-
else
|
72
|
-
attributes = attributes.nil? ? '' : ' ' + attributes.map { |(key, value)| "#{key}=\"#{value}\"" }.join(' ')
|
73
|
-
if content.nil?
|
74
|
-
"<#{tag}#{attributes} />"
|
75
|
-
else
|
137
|
+
def tag(tag, content = nil, attributes = nil)
|
138
|
+
if block_given?
|
139
|
+
attributes = content.nil? ? '' : ' ' + content.map { |(key, value)| "#{key}=\"#{value}\"" }.join(' ')
|
140
|
+
content_string = ''
|
141
|
+
content = yield(content_string)
|
142
|
+
content = content_string unless content_string.empty?
|
76
143
|
"<#{tag}#{attributes}>#{content}</#{tag}>"
|
144
|
+
else
|
145
|
+
attributes = attributes.nil? ? '' : ' ' + attributes.map { |(key, value)| "#{key}=\"#{value}\"" }.join(' ')
|
146
|
+
if content.nil?
|
147
|
+
"<#{tag}#{attributes} />"
|
148
|
+
else
|
149
|
+
if content.class == Float
|
150
|
+
"<#{tag}#{attributes}><div class='color_bar' style=\"width:#{(content*200).floor}px;\"/></#{tag}>"
|
151
|
+
else
|
152
|
+
"<#{tag}#{attributes}>#{content}</#{tag}>"
|
153
|
+
end
|
154
|
+
end
|
77
155
|
end
|
78
|
-
end
|
79
|
-
end
|
156
|
+
end
|
157
|
+
end
|
80
158
|
end
|
@@ -32,11 +32,13 @@ module RequestLogAnalyzer
|
|
32
32
|
|
33
33
|
# Adds another line to the request.
|
34
34
|
# The line should be provides as a hash of the fields parsed from the line.
|
35
|
-
def
|
35
|
+
def add_parsed_line (request_info_hash)
|
36
36
|
@lines << request_info_hash
|
37
37
|
@attributes = request_info_hash.merge(@attributes)
|
38
38
|
end
|
39
39
|
|
40
|
+
alias :<< :add_parsed_line
|
41
|
+
|
40
42
|
# Checks whether the given line type was parsed from the log file for this request
|
41
43
|
def has_line_type?(line_type)
|
42
44
|
return true if @lines.length == 1 && @lines[0][:line_type] == line_type.to_sym
|
@@ -9,7 +9,7 @@ module RequestLogAnalyzer::Source
|
|
9
9
|
# the log file simultaneously by different mongrel processes. This problem is detected by the
|
10
10
|
# parser, but the requests that are mixed up cannot be parsed. It will emit warnings when this
|
11
11
|
# occurs.
|
12
|
-
class
|
12
|
+
class LogParser < Base
|
13
13
|
|
14
14
|
attr_reader :source_files
|
15
15
|
|
@@ -22,6 +22,7 @@ module RequestLogAnalyzer::Source
|
|
22
22
|
@parsed_lines = 0
|
23
23
|
@parsed_requests = 0
|
24
24
|
@skipped_lines = 0
|
25
|
+
@skipped_requests = 0
|
25
26
|
@current_io = nil
|
26
27
|
@source_files = options[:source_files]
|
27
28
|
|
@@ -30,7 +31,7 @@ module RequestLogAnalyzer::Source
|
|
30
31
|
self.register_file_format(format)
|
31
32
|
end
|
32
33
|
|
33
|
-
def
|
34
|
+
def each_request(options = {}, &block)
|
34
35
|
|
35
36
|
case @source_files
|
36
37
|
when IO;
|
@@ -131,22 +132,22 @@ module RequestLogAnalyzer::Source
|
|
131
132
|
unless @current_request.nil?
|
132
133
|
if options[:assume_correct_order]
|
133
134
|
@parsed_requests += 1
|
134
|
-
yield @current_request
|
135
|
-
@current_request =
|
135
|
+
handle_request(@current_request, &block) #yield @current_request
|
136
|
+
@current_request = @file_format.create_request(request_data)
|
136
137
|
else
|
137
138
|
@skipped_lines += 1
|
138
139
|
warn(:unclosed_request, "Encountered header line, but previous request was not closed!")
|
139
140
|
@current_request = nil # remove all data that was parsed, skip next request as well.
|
140
141
|
end
|
141
142
|
else
|
142
|
-
@current_request =
|
143
|
+
@current_request = @file_format.create_request(request_data)
|
143
144
|
end
|
144
145
|
else
|
145
146
|
unless @current_request.nil?
|
146
147
|
@current_request << request_data
|
147
148
|
if footer_line?(request_data)
|
148
149
|
@parsed_requests += 1
|
149
|
-
yield @current_request
|
150
|
+
handle_request(@current_request, &block) # yield @current_request
|
150
151
|
@current_request = nil
|
151
152
|
end
|
152
153
|
else
|
@@ -155,6 +156,14 @@ module RequestLogAnalyzer::Source
|
|
155
156
|
end
|
156
157
|
end
|
157
158
|
end
|
159
|
+
|
160
|
+
# Handles the parsed request by calling the request handler.
|
161
|
+
# The default controller will send the request to every running aggegator.
|
162
|
+
def handle_request(request, &block)
|
163
|
+
@parsed_requests += 1
|
164
|
+
accepted = block_given? ? yield(request) : true
|
165
|
+
@skipped_requests += 1 if not accepted
|
166
|
+
end
|
158
167
|
|
159
168
|
# Checks whether a given line hash is a header line.
|
160
169
|
def header_line?(hash)
|
@@ -0,0 +1,58 @@
|
|
1
|
+
module RequestLogAnalyzer::Tracker
|
2
|
+
|
3
|
+
# const_missing: this function is used to load subclasses in the RequestLogAnalyzer::Track namespace.
|
4
|
+
# It will automatically load the required file based on the class name
|
5
|
+
def self.const_missing(const)
|
6
|
+
RequestLogAnalyzer::load_default_class_file(self, const)
|
7
|
+
end
|
8
|
+
|
9
|
+
# Base Tracker class. All other trackers inherit from this class
|
10
|
+
#
|
11
|
+
# Accepts the following options:
|
12
|
+
# * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
|
13
|
+
# * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
|
14
|
+
# * <tt>:output</tt> Direct output here (defaults to STDOUT)
|
15
|
+
#
|
16
|
+
# For example :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
|
17
|
+
class Base
|
18
|
+
|
19
|
+
attr_reader :options
|
20
|
+
|
21
|
+
def initialize(options ={})
|
22
|
+
@options = options
|
23
|
+
end
|
24
|
+
|
25
|
+
def prepare
|
26
|
+
end
|
27
|
+
|
28
|
+
def update(request)
|
29
|
+
end
|
30
|
+
|
31
|
+
def finalize
|
32
|
+
end
|
33
|
+
|
34
|
+
def should_update?(request)
|
35
|
+
return false if options[:line_type] && !request.has_line_type?(options[:line_type])
|
36
|
+
|
37
|
+
if options[:if].kind_of?(Symbol)
|
38
|
+
return false unless request[options[:if]]
|
39
|
+
elsif options[:if].respond_to?(:call)
|
40
|
+
return false unless options[:if].call(request)
|
41
|
+
end
|
42
|
+
|
43
|
+
if options[:unless].kind_of?(Symbol)
|
44
|
+
return false if request[options[:unless]]
|
45
|
+
elsif options[:unless].respond_to?(:call)
|
46
|
+
return false if options[:unless].call(request)
|
47
|
+
end
|
48
|
+
|
49
|
+
return true
|
50
|
+
end
|
51
|
+
|
52
|
+
def report(output)
|
53
|
+
output << self.inspect
|
54
|
+
output << "\n"
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
58
|
+
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
module RequestLogAnalyzer::Tracker
|
2
|
-
|
2
|
+
|
3
3
|
# Catagorize requests.
|
4
4
|
# Count and analyze requests for a specific attribute
|
5
5
|
#
|
@@ -19,10 +19,10 @@ module RequestLogAnalyzer::Tracker
|
|
19
19
|
# PUT | 13685 hits (28.4%) |░░░░░░░░░░░
|
20
20
|
# POST | 11662 hits (24.2%) |░░░░░░░░░
|
21
21
|
# DELETE | 512 hits (1.1%) |
|
22
|
-
class Category <
|
23
|
-
|
22
|
+
class Category < Base
|
23
|
+
|
24
24
|
attr_reader :categories
|
25
|
-
|
25
|
+
|
26
26
|
def prepare
|
27
27
|
raise "No categorizer set up for category tracker #{self.inspect}" unless options[:category]
|
28
28
|
@categories = {}
|
@@ -30,7 +30,7 @@ module RequestLogAnalyzer::Tracker
|
|
30
30
|
options[:all_categories].each { |cat| @categories[cat] = 0 }
|
31
31
|
end
|
32
32
|
end
|
33
|
-
|
33
|
+
|
34
34
|
def update(request)
|
35
35
|
cat = options[:category].respond_to?(:call) ? options[:category].call(request) : request[options[:category]]
|
36
36
|
if !cat.nil? || options[:nils]
|
@@ -38,10 +38,10 @@ module RequestLogAnalyzer::Tracker
|
|
38
38
|
@categories[cat] += 1
|
39
39
|
end
|
40
40
|
end
|
41
|
-
|
41
|
+
|
42
42
|
def report(output)
|
43
43
|
output.title(options[:title]) if options[:title]
|
44
|
-
|
44
|
+
|
45
45
|
if @categories.empty?
|
46
46
|
output << "None found.\n"
|
47
47
|
else
|
@@ -57,6 +57,5 @@ module RequestLogAnalyzer::Tracker
|
|
57
57
|
|
58
58
|
end
|
59
59
|
end
|
60
|
-
|
61
60
|
end
|
62
61
|
end
|
@@ -19,46 +19,49 @@ module RequestLogAnalyzer::Tracker
|
|
19
19
|
# EmployeeController#update.html [POST] | 4647 | 2731.23s | 0.59s
|
20
20
|
# EmployeeController#index.html [GET] | 5802 | 1477.32s | 0.25s
|
21
21
|
# .............
|
22
|
-
class Duration <
|
22
|
+
class Duration < Base
|
23
|
+
|
23
24
|
attr_reader :categories
|
24
|
-
|
25
|
+
|
25
26
|
def prepare
|
26
27
|
raise "No duration field set up for category tracker #{self.inspect}" unless options[:duration]
|
27
28
|
raise "No categorizer set up for duration tracker #{self.inspect}" unless options[:category]
|
28
|
-
|
29
|
+
|
29
30
|
@categories = {}
|
30
31
|
end
|
31
|
-
|
32
|
+
|
32
33
|
def update(request)
|
33
34
|
category = options[:category].respond_to?(:call) ? options[:category].call(request) : request[options[:category]]
|
34
35
|
duration = options[:duration].respond_to?(:call) ? options[:duration].call(request) : request[options[:duration]]
|
35
|
-
|
36
|
+
|
36
37
|
if !duration.nil? && !category.nil?
|
37
38
|
@categories[category] ||= {:count => 0, :total_duration => 0.0}
|
38
39
|
@categories[category][:count] += 1
|
39
40
|
@categories[category][:total_duration] += duration
|
40
41
|
end
|
41
42
|
end
|
42
|
-
|
43
|
+
|
43
44
|
def report_table(output, amount = 10, options = {}, &block)
|
44
|
-
|
45
|
+
|
46
|
+
output.title(options[:title])
|
47
|
+
|
45
48
|
top_categories = @categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }.slice(0...amount)
|
46
|
-
output.table({:title =>
|
49
|
+
output.table({:title => 'Action'}, {:title => 'Hits', :align => :right, :min_width => 4},
|
47
50
|
{:title => 'Cumulative', :align => :right, :min_width => 10}, {:title => 'Average', :align => :right, :min_width => 8}) do |rows|
|
48
|
-
|
51
|
+
|
49
52
|
top_categories.each do |(cat, info)|
|
50
53
|
rows << [cat, info[:count], "%0.02fs" % info[:total_duration], "%0.02fs" % (info[:total_duration] / info[:count])]
|
51
54
|
end
|
52
55
|
end
|
53
56
|
|
54
57
|
end
|
55
|
-
|
58
|
+
|
56
59
|
def report(output)
|
57
60
|
|
58
61
|
options[:title] ||= 'Request duration'
|
59
62
|
options[:report] ||= [:total, :average]
|
60
63
|
options[:top] ||= 20
|
61
|
-
|
64
|
+
|
62
65
|
options[:report].each do |report|
|
63
66
|
case report
|
64
67
|
when :average
|
@@ -73,4 +76,4 @@ module RequestLogAnalyzer::Tracker
|
|
73
76
|
end
|
74
77
|
end
|
75
78
|
end
|
76
|
-
end
|
79
|
+
end
|