request-log-analyzer 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +4 -3
- data/bin/request-log-analyzer +4 -5
- data/lib/cli/command_line_arguments.rb +2 -2
- data/lib/request_log_analyzer.rb +28 -6
- data/lib/request_log_analyzer/{aggregator/base.rb → aggregator.rb} +5 -1
- data/lib/request_log_analyzer/aggregator/summarizer.rb +2 -3
- data/lib/request_log_analyzer/controller.rb +11 -16
- data/lib/request_log_analyzer/file_format.rb +71 -38
- data/lib/request_log_analyzer/file_format/merb.rb +32 -26
- data/lib/request_log_analyzer/file_format/rails.rb +73 -71
- data/lib/request_log_analyzer/file_format/rails_development.rb +93 -95
- data/lib/request_log_analyzer/filter.rb +38 -0
- data/lib/request_log_analyzer/filter/anonimize.rb +1 -1
- data/lib/request_log_analyzer/line_definition.rb +1 -1
- data/lib/request_log_analyzer/output.rb +6 -8
- data/lib/request_log_analyzer/output/fixed_width.rb +133 -117
- data/lib/request_log_analyzer/output/html.rb +138 -60
- data/lib/request_log_analyzer/request.rb +3 -1
- data/lib/request_log_analyzer/{source/base.rb → source.rb} +5 -0
- data/lib/request_log_analyzer/source/{log_file.rb → log_parser.rb} +15 -6
- data/lib/request_log_analyzer/tracker.rb +58 -0
- data/lib/request_log_analyzer/tracker/category.rb +7 -8
- data/lib/request_log_analyzer/tracker/duration.rb +15 -12
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -8
- data/lib/request_log_analyzer/tracker/timespan.rb +10 -10
- data/spec/controller_spec.rb +5 -4
- data/spec/database_inserter_spec.rb +5 -8
- data/spec/file_format_spec.rb +2 -2
- data/spec/file_formats/spec_format.rb +2 -1
- data/spec/filter_spec.rb +0 -3
- data/spec/log_parser_spec.rb +6 -6
- data/spec/merb_format_spec.rb +38 -38
- data/spec/rails_format_spec.rb +2 -2
- data/spec/request_spec.rb +2 -2
- data/spec/spec_helper.rb +3 -37
- data/tasks/github-gem.rake +2 -1
- metadata +7 -8
- data/lib/request_log_analyzer/filter/base.rb +0 -32
- data/lib/request_log_analyzer/log_parser.rb +0 -173
- data/lib/request_log_analyzer/tracker/base.rb +0 -54
@@ -1,80 +1,158 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
1
|
+
module RequestLogAnalyzer::Output
|
2
|
+
|
3
|
+
class HTML < Base
|
4
|
+
|
5
|
+
# def initialize(io, options = {})
|
6
|
+
# super(io, options)
|
7
|
+
# end
|
8
|
+
|
9
|
+
def print(str)
|
10
|
+
@io << str
|
11
|
+
end
|
10
12
|
|
11
|
-
|
13
|
+
alias :<< :print
|
12
14
|
|
13
|
-
|
14
|
-
|
15
|
-
|
15
|
+
def puts(str = '')
|
16
|
+
@io << str << "<br/>\n"
|
17
|
+
end
|
16
18
|
|
17
|
-
|
18
|
-
|
19
|
-
|
19
|
+
def title(title)
|
20
|
+
@io.puts(tag(:h2, title))
|
21
|
+
end
|
20
22
|
|
21
|
-
|
22
|
-
|
23
|
-
|
23
|
+
def line(*font)
|
24
|
+
@io.puts(tag(:hr))
|
25
|
+
end
|
24
26
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
27
|
+
def link(text, url = nil)
|
28
|
+
url = text if url.nil?
|
29
|
+
tag(:a, text, :href => url)
|
30
|
+
end
|
29
31
|
|
30
|
-
|
31
|
-
|
32
|
-
|
32
|
+
def table(*columns, &block)
|
33
|
+
rows = Array.new
|
34
|
+
yield(rows)
|
33
35
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
36
|
+
@io << tag(:table, {:id => 'mytable', :cellspacing => 0}) do |content|
|
37
|
+
if table_has_header?(columns)
|
38
|
+
content << tag(:tr) do
|
39
|
+
columns.map { |col| tag(:th, col[:title]) }.join("\n")
|
40
|
+
end
|
38
41
|
end
|
39
|
-
end
|
40
42
|
|
41
|
-
|
42
|
-
|
43
|
-
|
43
|
+
odd = false
|
44
|
+
rows.each do |row|
|
45
|
+
odd = !odd
|
46
|
+
content << tag(:tr) do
|
47
|
+
if odd
|
48
|
+
row.map { |cell| tag(:td, cell, :class => 'alt') }.join("\n")
|
49
|
+
else
|
50
|
+
row.map { |cell| tag(:td, cell) }.join("\n")
|
51
|
+
end
|
52
|
+
end
|
44
53
|
end
|
45
54
|
end
|
46
|
-
end
|
47
55
|
|
48
|
-
|
56
|
+
end
|
49
57
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
58
|
+
def header
|
59
|
+
@io << "<html>"
|
60
|
+
@io << tag(:head) do |headers|
|
61
|
+
headers << tag(:title, 'Request-log-analyzer report')
|
62
|
+
headers << tag(:style, '
|
63
|
+
body {
|
64
|
+
font: normal 11px auto "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
65
|
+
color: #4f6b72;
|
66
|
+
background: #E6EAE9;
|
67
|
+
padding-left:20px;
|
68
|
+
padding-top:20px;
|
69
|
+
padding-bottom:20px;
|
70
|
+
}
|
71
|
+
|
72
|
+
a {
|
73
|
+
color: #c75f3e;
|
74
|
+
}
|
75
|
+
|
76
|
+
.color_bar {
|
77
|
+
border: 1px solid;
|
78
|
+
height:10px;
|
79
|
+
background: #CAE8EA;
|
80
|
+
}
|
81
|
+
|
82
|
+
#mytable {
|
83
|
+
width: 700px;
|
84
|
+
padding: 0;
|
85
|
+
margin: 0;
|
86
|
+
padding-bottom:10px;
|
87
|
+
}
|
88
|
+
|
89
|
+
caption {
|
90
|
+
padding: 0 0 5px 0;
|
91
|
+
width: 700px;
|
92
|
+
font: italic 11px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
93
|
+
text-align: right;
|
94
|
+
}
|
95
|
+
|
96
|
+
th {
|
97
|
+
font: bold 11px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
98
|
+
color: #4f6b72;
|
99
|
+
border-right: 1px solid #C1DAD7;
|
100
|
+
border-bottom: 1px solid #C1DAD7;
|
101
|
+
border-top: 1px solid #C1DAD7;
|
102
|
+
letter-spacing: 2px;
|
103
|
+
text-transform: uppercase;
|
104
|
+
text-align: left;
|
105
|
+
padding: 6px 6px 6px 12px;
|
106
|
+
background: #CAE8EA url(images/bg_header.jpg) no-repeat;
|
107
|
+
}
|
108
|
+
|
109
|
+
td {
|
110
|
+
font: bold 11px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
111
|
+
border-right: 1px solid #C1DAD7;
|
112
|
+
border-bottom: 1px solid #C1DAD7;
|
113
|
+
background: #fff;
|
114
|
+
padding: 6px 6px 6px 12px;
|
115
|
+
color: #4f6b72;
|
116
|
+
}
|
117
|
+
|
118
|
+
td.alt {
|
119
|
+
background: #F5FAFA;
|
120
|
+
color: #797268;
|
121
|
+
}
|
122
|
+
', :type => "text/css")
|
123
|
+
end
|
124
|
+
@io << '<body>'
|
125
|
+
@io << tag(:h1, 'Request-log-analyzer summary report')
|
126
|
+
@io << tag(:p, "Version #{RequestLogAnalyzer::VERSION} - written by Willem van Bergen and Bart ten Brinke")
|
54
127
|
end
|
55
|
-
@io << '<body>'
|
56
|
-
end
|
57
128
|
|
58
|
-
|
59
|
-
|
60
|
-
|
129
|
+
def footer
|
130
|
+
@io << tag(:hr) << tag(:h2, 'Thanks for using request-log-analyzer')
|
131
|
+
@io << tag(:p, 'Please visit the ' + link('Request-log-analyzer website', 'http://github.com/wvanbergen/request-log-analyzer'))
|
132
|
+
@io << "</body></html>\n"
|
133
|
+
end
|
61
134
|
|
62
|
-
|
135
|
+
protected
|
63
136
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
"<#{tag}#{attributes}>#{content}</#{tag}>"
|
71
|
-
else
|
72
|
-
attributes = attributes.nil? ? '' : ' ' + attributes.map { |(key, value)| "#{key}=\"#{value}\"" }.join(' ')
|
73
|
-
if content.nil?
|
74
|
-
"<#{tag}#{attributes} />"
|
75
|
-
else
|
137
|
+
def tag(tag, content = nil, attributes = nil)
|
138
|
+
if block_given?
|
139
|
+
attributes = content.nil? ? '' : ' ' + content.map { |(key, value)| "#{key}=\"#{value}\"" }.join(' ')
|
140
|
+
content_string = ''
|
141
|
+
content = yield(content_string)
|
142
|
+
content = content_string unless content_string.empty?
|
76
143
|
"<#{tag}#{attributes}>#{content}</#{tag}>"
|
144
|
+
else
|
145
|
+
attributes = attributes.nil? ? '' : ' ' + attributes.map { |(key, value)| "#{key}=\"#{value}\"" }.join(' ')
|
146
|
+
if content.nil?
|
147
|
+
"<#{tag}#{attributes} />"
|
148
|
+
else
|
149
|
+
if content.class == Float
|
150
|
+
"<#{tag}#{attributes}><div class='color_bar' style=\"width:#{(content*200).floor}px;\"/></#{tag}>"
|
151
|
+
else
|
152
|
+
"<#{tag}#{attributes}>#{content}</#{tag}>"
|
153
|
+
end
|
154
|
+
end
|
77
155
|
end
|
78
|
-
end
|
79
|
-
end
|
156
|
+
end
|
157
|
+
end
|
80
158
|
end
|
@@ -32,11 +32,13 @@ module RequestLogAnalyzer
|
|
32
32
|
|
33
33
|
# Adds another line to the request.
|
34
34
|
# The line should be provides as a hash of the fields parsed from the line.
|
35
|
-
def
|
35
|
+
def add_parsed_line (request_info_hash)
|
36
36
|
@lines << request_info_hash
|
37
37
|
@attributes = request_info_hash.merge(@attributes)
|
38
38
|
end
|
39
39
|
|
40
|
+
alias :<< :add_parsed_line
|
41
|
+
|
40
42
|
# Checks whether the given line type was parsed from the log file for this request
|
41
43
|
def has_line_type?(line_type)
|
42
44
|
return true if @lines.length == 1 && @lines[0][:line_type] == line_type.to_sym
|
@@ -9,7 +9,7 @@ module RequestLogAnalyzer::Source
|
|
9
9
|
# the log file simultaneously by different mongrel processes. This problem is detected by the
|
10
10
|
# parser, but the requests that are mixed up cannot be parsed. It will emit warnings when this
|
11
11
|
# occurs.
|
12
|
-
class
|
12
|
+
class LogParser < Base
|
13
13
|
|
14
14
|
attr_reader :source_files
|
15
15
|
|
@@ -22,6 +22,7 @@ module RequestLogAnalyzer::Source
|
|
22
22
|
@parsed_lines = 0
|
23
23
|
@parsed_requests = 0
|
24
24
|
@skipped_lines = 0
|
25
|
+
@skipped_requests = 0
|
25
26
|
@current_io = nil
|
26
27
|
@source_files = options[:source_files]
|
27
28
|
|
@@ -30,7 +31,7 @@ module RequestLogAnalyzer::Source
|
|
30
31
|
self.register_file_format(format)
|
31
32
|
end
|
32
33
|
|
33
|
-
def
|
34
|
+
def each_request(options = {}, &block)
|
34
35
|
|
35
36
|
case @source_files
|
36
37
|
when IO;
|
@@ -131,22 +132,22 @@ module RequestLogAnalyzer::Source
|
|
131
132
|
unless @current_request.nil?
|
132
133
|
if options[:assume_correct_order]
|
133
134
|
@parsed_requests += 1
|
134
|
-
yield @current_request
|
135
|
-
@current_request =
|
135
|
+
handle_request(@current_request, &block) #yield @current_request
|
136
|
+
@current_request = @file_format.create_request(request_data)
|
136
137
|
else
|
137
138
|
@skipped_lines += 1
|
138
139
|
warn(:unclosed_request, "Encountered header line, but previous request was not closed!")
|
139
140
|
@current_request = nil # remove all data that was parsed, skip next request as well.
|
140
141
|
end
|
141
142
|
else
|
142
|
-
@current_request =
|
143
|
+
@current_request = @file_format.create_request(request_data)
|
143
144
|
end
|
144
145
|
else
|
145
146
|
unless @current_request.nil?
|
146
147
|
@current_request << request_data
|
147
148
|
if footer_line?(request_data)
|
148
149
|
@parsed_requests += 1
|
149
|
-
yield @current_request
|
150
|
+
handle_request(@current_request, &block) # yield @current_request
|
150
151
|
@current_request = nil
|
151
152
|
end
|
152
153
|
else
|
@@ -155,6 +156,14 @@ module RequestLogAnalyzer::Source
|
|
155
156
|
end
|
156
157
|
end
|
157
158
|
end
|
159
|
+
|
160
|
+
# Handles the parsed request by calling the request handler.
|
161
|
+
# The default controller will send the request to every running aggegator.
|
162
|
+
def handle_request(request, &block)
|
163
|
+
@parsed_requests += 1
|
164
|
+
accepted = block_given? ? yield(request) : true
|
165
|
+
@skipped_requests += 1 if not accepted
|
166
|
+
end
|
158
167
|
|
159
168
|
# Checks whether a given line hash is a header line.
|
160
169
|
def header_line?(hash)
|
@@ -0,0 +1,58 @@
|
|
1
|
+
module RequestLogAnalyzer::Tracker
|
2
|
+
|
3
|
+
# const_missing: this function is used to load subclasses in the RequestLogAnalyzer::Track namespace.
|
4
|
+
# It will automatically load the required file based on the class name
|
5
|
+
def self.const_missing(const)
|
6
|
+
RequestLogAnalyzer::load_default_class_file(self, const)
|
7
|
+
end
|
8
|
+
|
9
|
+
# Base Tracker class. All other trackers inherit from this class
|
10
|
+
#
|
11
|
+
# Accepts the following options:
|
12
|
+
# * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
|
13
|
+
# * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
|
14
|
+
# * <tt>:output</tt> Direct output here (defaults to STDOUT)
|
15
|
+
#
|
16
|
+
# For example :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
|
17
|
+
class Base
|
18
|
+
|
19
|
+
attr_reader :options
|
20
|
+
|
21
|
+
def initialize(options ={})
|
22
|
+
@options = options
|
23
|
+
end
|
24
|
+
|
25
|
+
def prepare
|
26
|
+
end
|
27
|
+
|
28
|
+
def update(request)
|
29
|
+
end
|
30
|
+
|
31
|
+
def finalize
|
32
|
+
end
|
33
|
+
|
34
|
+
def should_update?(request)
|
35
|
+
return false if options[:line_type] && !request.has_line_type?(options[:line_type])
|
36
|
+
|
37
|
+
if options[:if].kind_of?(Symbol)
|
38
|
+
return false unless request[options[:if]]
|
39
|
+
elsif options[:if].respond_to?(:call)
|
40
|
+
return false unless options[:if].call(request)
|
41
|
+
end
|
42
|
+
|
43
|
+
if options[:unless].kind_of?(Symbol)
|
44
|
+
return false if request[options[:unless]]
|
45
|
+
elsif options[:unless].respond_to?(:call)
|
46
|
+
return false if options[:unless].call(request)
|
47
|
+
end
|
48
|
+
|
49
|
+
return true
|
50
|
+
end
|
51
|
+
|
52
|
+
def report(output)
|
53
|
+
output << self.inspect
|
54
|
+
output << "\n"
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
58
|
+
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
module RequestLogAnalyzer::Tracker
|
2
|
-
|
2
|
+
|
3
3
|
# Catagorize requests.
|
4
4
|
# Count and analyze requests for a specific attribute
|
5
5
|
#
|
@@ -19,10 +19,10 @@ module RequestLogAnalyzer::Tracker
|
|
19
19
|
# PUT | 13685 hits (28.4%) |░░░░░░░░░░░
|
20
20
|
# POST | 11662 hits (24.2%) |░░░░░░░░░
|
21
21
|
# DELETE | 512 hits (1.1%) |
|
22
|
-
class Category <
|
23
|
-
|
22
|
+
class Category < Base
|
23
|
+
|
24
24
|
attr_reader :categories
|
25
|
-
|
25
|
+
|
26
26
|
def prepare
|
27
27
|
raise "No categorizer set up for category tracker #{self.inspect}" unless options[:category]
|
28
28
|
@categories = {}
|
@@ -30,7 +30,7 @@ module RequestLogAnalyzer::Tracker
|
|
30
30
|
options[:all_categories].each { |cat| @categories[cat] = 0 }
|
31
31
|
end
|
32
32
|
end
|
33
|
-
|
33
|
+
|
34
34
|
def update(request)
|
35
35
|
cat = options[:category].respond_to?(:call) ? options[:category].call(request) : request[options[:category]]
|
36
36
|
if !cat.nil? || options[:nils]
|
@@ -38,10 +38,10 @@ module RequestLogAnalyzer::Tracker
|
|
38
38
|
@categories[cat] += 1
|
39
39
|
end
|
40
40
|
end
|
41
|
-
|
41
|
+
|
42
42
|
def report(output)
|
43
43
|
output.title(options[:title]) if options[:title]
|
44
|
-
|
44
|
+
|
45
45
|
if @categories.empty?
|
46
46
|
output << "None found.\n"
|
47
47
|
else
|
@@ -57,6 +57,5 @@ module RequestLogAnalyzer::Tracker
|
|
57
57
|
|
58
58
|
end
|
59
59
|
end
|
60
|
-
|
61
60
|
end
|
62
61
|
end
|
@@ -19,46 +19,49 @@ module RequestLogAnalyzer::Tracker
|
|
19
19
|
# EmployeeController#update.html [POST] | 4647 | 2731.23s | 0.59s
|
20
20
|
# EmployeeController#index.html [GET] | 5802 | 1477.32s | 0.25s
|
21
21
|
# .............
|
22
|
-
class Duration <
|
22
|
+
class Duration < Base
|
23
|
+
|
23
24
|
attr_reader :categories
|
24
|
-
|
25
|
+
|
25
26
|
def prepare
|
26
27
|
raise "No duration field set up for category tracker #{self.inspect}" unless options[:duration]
|
27
28
|
raise "No categorizer set up for duration tracker #{self.inspect}" unless options[:category]
|
28
|
-
|
29
|
+
|
29
30
|
@categories = {}
|
30
31
|
end
|
31
|
-
|
32
|
+
|
32
33
|
def update(request)
|
33
34
|
category = options[:category].respond_to?(:call) ? options[:category].call(request) : request[options[:category]]
|
34
35
|
duration = options[:duration].respond_to?(:call) ? options[:duration].call(request) : request[options[:duration]]
|
35
|
-
|
36
|
+
|
36
37
|
if !duration.nil? && !category.nil?
|
37
38
|
@categories[category] ||= {:count => 0, :total_duration => 0.0}
|
38
39
|
@categories[category][:count] += 1
|
39
40
|
@categories[category][:total_duration] += duration
|
40
41
|
end
|
41
42
|
end
|
42
|
-
|
43
|
+
|
43
44
|
def report_table(output, amount = 10, options = {}, &block)
|
44
|
-
|
45
|
+
|
46
|
+
output.title(options[:title])
|
47
|
+
|
45
48
|
top_categories = @categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }.slice(0...amount)
|
46
|
-
output.table({:title =>
|
49
|
+
output.table({:title => 'Action'}, {:title => 'Hits', :align => :right, :min_width => 4},
|
47
50
|
{:title => 'Cumulative', :align => :right, :min_width => 10}, {:title => 'Average', :align => :right, :min_width => 8}) do |rows|
|
48
|
-
|
51
|
+
|
49
52
|
top_categories.each do |(cat, info)|
|
50
53
|
rows << [cat, info[:count], "%0.02fs" % info[:total_duration], "%0.02fs" % (info[:total_duration] / info[:count])]
|
51
54
|
end
|
52
55
|
end
|
53
56
|
|
54
57
|
end
|
55
|
-
|
58
|
+
|
56
59
|
def report(output)
|
57
60
|
|
58
61
|
options[:title] ||= 'Request duration'
|
59
62
|
options[:report] ||= [:total, :average]
|
60
63
|
options[:top] ||= 20
|
61
|
-
|
64
|
+
|
62
65
|
options[:report].each do |report|
|
63
66
|
case report
|
64
67
|
when :average
|
@@ -73,4 +76,4 @@ module RequestLogAnalyzer::Tracker
|
|
73
76
|
end
|
74
77
|
end
|
75
78
|
end
|
76
|
-
end
|
79
|
+
end
|