request-log-analyzer 1.5.2 → 1.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/request-log-analyzer +9 -5
- data/lib/request_log_analyzer.rb +1 -1
- data/lib/request_log_analyzer/aggregator/echo.rb +2 -1
- data/lib/request_log_analyzer/aggregator/summarizer.rb +5 -37
- data/lib/request_log_analyzer/controller.rb +4 -3
- data/lib/request_log_analyzer/file_format.rb +68 -1
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +3 -1
- data/lib/request_log_analyzer/file_format/apache.rb +7 -2
- data/lib/request_log_analyzer/file_format/merb.rb +3 -1
- data/lib/request_log_analyzer/file_format/mysql.rb +10 -5
- data/lib/request_log_analyzer/file_format/rack.rb +1 -3
- data/lib/request_log_analyzer/file_format/rails.rb +5 -5
- data/lib/request_log_analyzer/output.rb +4 -0
- data/lib/request_log_analyzer/output/fancy_html.rb +31 -0
- data/lib/request_log_analyzer/tracker.rb +0 -105
- data/lib/request_log_analyzer/tracker/duration.rb +3 -62
- data/lib/request_log_analyzer/tracker/frequency.rb +18 -5
- data/lib/request_log_analyzer/tracker/numeric_value.rb +223 -0
- data/lib/request_log_analyzer/tracker/traffic.rb +6 -72
- data/request-log-analyzer.gemspec +4 -4
- data/spec/lib/mocks.rb +1 -0
- data/spec/unit/file_format/common_regular_expressions_spec.rb +53 -0
- data/spec/unit/file_format/mysql_format_spec.rb +1 -2
- data/spec/unit/file_format/rails_format_spec.rb +9 -4
- data/spec/unit/tracker/duration_tracker_spec.rb +2 -83
- data/spec/unit/tracker/{count_tracker_spec.rb → numeric_value_tracker_spec.rb} +50 -7
- data/spec/unit/tracker/tracker_api_spec.rb +3 -2
- data/spec/unit/tracker/traffic_tracker_spec.rb +0 -79
- metadata +8 -5
- data/lib/request_log_analyzer/tracker/count.rb +0 -93
@@ -19,54 +19,12 @@ module RequestLogAnalyzer::Tracker
|
|
19
19
|
# EmployeeController#update.html [POST] | 4647 | 2731.23s | 0.59s
|
20
20
|
# EmployeeController#index.html [GET] | 5802 | 1477.32s | 0.25s
|
21
21
|
# .............
|
22
|
-
class Duration <
|
23
|
-
|
24
|
-
include RequestLogAnalyzer::Tracker::StatisticsTracking
|
25
|
-
|
26
|
-
attr_reader :categories
|
22
|
+
class Duration < NumericValue
|
27
23
|
|
28
24
|
# Check if duration and catagory option have been received,
|
29
25
|
def prepare
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
unless options[:multiple]
|
34
|
-
@categorizer = create_lambda(options[:category])
|
35
|
-
@durationizer = create_lambda(options[:duration])
|
36
|
-
end
|
37
|
-
|
38
|
-
@categories = {}
|
39
|
-
end
|
40
|
-
|
41
|
-
# Get the duration information fron the request and store it in the different categories.
|
42
|
-
# <tt>request</tt> The request.
|
43
|
-
def update(request)
|
44
|
-
if options[:multiple]
|
45
|
-
found_categories = request.every(options[:category])
|
46
|
-
found_durations = request.every(options[:duration])
|
47
|
-
raise "Capture mismatch for multiple values in a request" unless found_categories.length == found_durations.length
|
48
|
-
found_categories.each_with_index { |cat, index| update_statistics(cat, found_durations[index]) }
|
49
|
-
else
|
50
|
-
category = @categorizer.call(request)
|
51
|
-
duration = @durationizer.call(request)
|
52
|
-
update_statistics(category, duration) if duration.kind_of?(Numeric) && category
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
# Block function to build a result table using a provided sorting function.
|
57
|
-
# <tt>output</tt> The output object.
|
58
|
-
# <tt>amount</tt> The number of rows in the report table (default 10).
|
59
|
-
# === Options
|
60
|
-
# * </tt>:title</tt> The title of the table
|
61
|
-
# * </tt>:sort</tt> The key to sort on (:hits, :cumulative, :average, :min or :max)
|
62
|
-
def report_table(output, sort, options = {}, &block)
|
63
|
-
output.puts
|
64
|
-
top_categories = output.slice_results(sorted_by(sort))
|
65
|
-
output.with_style(:top_line => true) do
|
66
|
-
output.table(*statistics_header(:title => options[:title], :highlight => sort)) do |rows|
|
67
|
-
top_categories.each { |(cat, info)| rows << statistics_row(cat) }
|
68
|
-
end
|
69
|
-
end
|
26
|
+
options[:value] = options[:duration] if options[:duration]
|
27
|
+
super
|
70
28
|
end
|
71
29
|
|
72
30
|
# Display a duration
|
@@ -79,26 +37,9 @@ module RequestLogAnalyzer::Tracker
|
|
79
37
|
end
|
80
38
|
end
|
81
39
|
|
82
|
-
# Generate a request duration report to the given output object
|
83
|
-
# By default colulative and average duration are generated.
|
84
|
-
# Any options for the report should have been set during initialize.
|
85
|
-
# <tt>output</tt> The output object
|
86
|
-
def report(output)
|
87
|
-
sortings = output.options[:sort] || [:sum, :mean]
|
88
|
-
sortings.each do |sorting|
|
89
|
-
report_table(output, sorting, :title => "#{title} - by #{sorting}")
|
90
|
-
end
|
91
|
-
end
|
92
|
-
|
93
40
|
# Returns the title of this tracker for reports
|
94
41
|
def title
|
95
42
|
options[:title] || 'Request duration'
|
96
43
|
end
|
97
|
-
|
98
|
-
# Returns all the categories and the tracked duration as a hash than can be exported to YAML
|
99
|
-
def to_yaml_object
|
100
|
-
return nil if @categories.empty?
|
101
|
-
@categories
|
102
|
-
end
|
103
44
|
end
|
104
45
|
end
|
@@ -26,8 +26,10 @@ module RequestLogAnalyzer::Tracker
|
|
26
26
|
|
27
27
|
# Check if categories are set up
|
28
28
|
def prepare
|
29
|
+
options[:category] = options[:value] if options[:value] && !options[:category]
|
29
30
|
raise "No categorizer set up for category tracker #{self.inspect}" unless options[:category]
|
30
|
-
|
31
|
+
|
32
|
+
@categorizer = create_lambda(options[:category]) unless options[:multiple]
|
31
33
|
|
32
34
|
# Initialize the categories. Use the list of category names to
|
33
35
|
@categories = {}
|
@@ -37,10 +39,21 @@ module RequestLogAnalyzer::Tracker
|
|
37
39
|
# Check HTTP method of a request and store that in the categories hash.
|
38
40
|
# <tt>request</tt> The request.
|
39
41
|
def update(request)
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
42
|
+
if options[:multiple]
|
43
|
+
cats = request.every(options[:category])
|
44
|
+
cats.each do |cat|
|
45
|
+
if cat || options[:nils]
|
46
|
+
@categories[cat] ||= 0
|
47
|
+
@categories[cat] += 1
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
else
|
52
|
+
cat = @categorizer.call(request)
|
53
|
+
if cat || options[:nils]
|
54
|
+
@categories[cat] ||= 0
|
55
|
+
@categories[cat] += 1
|
56
|
+
end
|
44
57
|
end
|
45
58
|
end
|
46
59
|
|
@@ -0,0 +1,223 @@
|
|
1
|
+
module RequestLogAnalyzer::Tracker
|
2
|
+
|
3
|
+
class NumericValue < Base
|
4
|
+
|
5
|
+
attr_reader :categories
|
6
|
+
|
7
|
+
# Sets up the numeric value tracker. It will check whether the value and category
|
8
|
+
# options are set that are used to extract and categorize the values during
|
9
|
+
# parsing. Two lambda procedures are created for these tasks
|
10
|
+
def prepare
|
11
|
+
|
12
|
+
raise "No value field set up for numeric tracker #{self.inspect}" unless options[:value]
|
13
|
+
raise "No categorizer set up for numeric tracker #{self.inspect}" unless options[:category]
|
14
|
+
|
15
|
+
unless options[:multiple]
|
16
|
+
@categorizer = create_lambda(options[:category])
|
17
|
+
@valueizer = create_lambda(options[:value])
|
18
|
+
end
|
19
|
+
|
20
|
+
@categories = {}
|
21
|
+
end
|
22
|
+
|
23
|
+
# Get the value information from the request and store it in the respective categories.
|
24
|
+
#
|
25
|
+
# If a request can contain multiple usable values for this tracker, the :multiple option
|
26
|
+
# should be set to true. In this case, all the values and respective categories will be
|
27
|
+
# read from the request using the #every method from the fields given in the :value and
|
28
|
+
# :category option.
|
29
|
+
#
|
30
|
+
# If the request contains only one suitable value and the :multiple is not set, it will
|
31
|
+
# read the single value and category from the fields provided in the :value and :category
|
32
|
+
# option, or calculate it with any lambda procedure that is assigned to these options. The
|
33
|
+
# request will be passed to procedure as input for the calculation.
|
34
|
+
#
|
35
|
+
# @param [RequestLogAnalyzer::Request] request The request to get the information from.
|
36
|
+
def update(request)
|
37
|
+
if options[:multiple]
|
38
|
+
found_categories = request.every(options[:category])
|
39
|
+
found_values = request.every(options[:value])
|
40
|
+
raise "Capture mismatch for multiple values in a request" unless found_categories.length == found_values.length
|
41
|
+
|
42
|
+
found_categories.each_with_index do |cat, index|
|
43
|
+
update_statistics(cat, found_values[index]) if cat && found_values[index].kind_of?(Numeric)
|
44
|
+
end
|
45
|
+
|
46
|
+
else
|
47
|
+
category = @categorizer.call(request)
|
48
|
+
value = @valueizer.call(request)
|
49
|
+
update_statistics(category, value) if value.kind_of?(Numeric) && category
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
# Block function to build a result table using a provided sorting function.
|
54
|
+
# <tt>output</tt> The output object.
|
55
|
+
# <tt>amount</tt> The number of rows in the report table (default 10).
|
56
|
+
# === Options
|
57
|
+
# * </tt>:title</tt> The title of the table
|
58
|
+
# * </tt>:sort</tt> The key to sort on (:hits, :cumulative, :average, :min or :max)
|
59
|
+
def report_table(output, sort, options = {}, &block)
|
60
|
+
output.puts
|
61
|
+
top_categories = output.slice_results(sorted_by(sort))
|
62
|
+
output.with_style(:top_line => true) do
|
63
|
+
output.table(*statistics_header(:title => options[:title], :highlight => sort)) do |rows|
|
64
|
+
top_categories.each { |(cat, info)| rows << statistics_row(cat) }
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
# Display a value
|
70
|
+
def display_value(value)
|
71
|
+
return "- " if value.nil?
|
72
|
+
return "0 " if value.zero?
|
73
|
+
|
74
|
+
case Math.log10(value).floor
|
75
|
+
when 0...4 then '%d ' % value
|
76
|
+
when 4...7 then '%dk' % (value / 1000)
|
77
|
+
when 7...10 then '%dM' % (value / 1000_000)
|
78
|
+
when 10...13 then '%dG' % (value / 1000_000_000)
|
79
|
+
when 13...16 then '%dT' % (value / 1000_000_000_000)
|
80
|
+
else '%dP' % (value / 1000_000_000_000_000)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
# Generate a request report to the given output object
|
85
|
+
# By default colulative and average duration are generated.
|
86
|
+
# Any options for the report should have been set during initialize.
|
87
|
+
# <tt>output</tt> The output object
|
88
|
+
def report(output)
|
89
|
+
sortings = output.options[:sort] || [:sum, :mean]
|
90
|
+
sortings.each do |sorting|
|
91
|
+
report_table(output, sorting, :title => "#{title} - by #{sorting}")
|
92
|
+
end
|
93
|
+
|
94
|
+
if options[:total]
|
95
|
+
output.puts
|
96
|
+
output.puts "#{output.colorize(title, :white, :bold)} - total: " + output.colorize(display_value(sum_overall), :brown, :bold)
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
# Returns the title of this tracker for reports
|
101
|
+
def title
|
102
|
+
@title ||= begin
|
103
|
+
if options[:title]
|
104
|
+
options[:title]
|
105
|
+
else
|
106
|
+
title_builder = ""
|
107
|
+
title_builder << "#{options[:value]} " if options[:value].kind_of?(Symbol)
|
108
|
+
title_builder << (options[:category].kind_of?(Symbol) ? "per #{options[:category]}" : "per request")
|
109
|
+
title_builder
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
# Returns all the categories and the tracked duration as a hash than can be exported to YAML
|
115
|
+
def to_yaml_object
|
116
|
+
return nil if @categories.empty?
|
117
|
+
@categories
|
118
|
+
end
|
119
|
+
|
120
|
+
|
121
|
+
# Update sthe running calculation of statistics with the newly found numeric value.
|
122
|
+
# <tt>category</tt>:: The category for which to update the running statistics calculations
|
123
|
+
# <tt>number</tt>:: The numeric value to update the calculations with.
|
124
|
+
def update_statistics(category, number)
|
125
|
+
@categories[category] ||= {:hits => 0, :sum => 0, :mean => 0.0, :sum_of_squares => 0.0, :min => number, :max => number }
|
126
|
+
delta = number - @categories[category][:mean]
|
127
|
+
|
128
|
+
@categories[category][:hits] += 1
|
129
|
+
@categories[category][:mean] += (delta / @categories[category][:hits])
|
130
|
+
@categories[category][:sum_of_squares] += delta * (number - @categories[category][:mean])
|
131
|
+
@categories[category][:sum] += number
|
132
|
+
@categories[category][:min] = number if number < @categories[category][:min]
|
133
|
+
@categories[category][:max] = number if number > @categories[category][:max]
|
134
|
+
end
|
135
|
+
|
136
|
+
# Get the number of hits of a specific category.
|
137
|
+
# <tt>cat</tt> The category
|
138
|
+
def hits(cat)
|
139
|
+
@categories[cat][:hits]
|
140
|
+
end
|
141
|
+
|
142
|
+
# Get the total duration of a specific category.
|
143
|
+
# <tt>cat</tt> The category
|
144
|
+
def sum(cat)
|
145
|
+
@categories[cat][:sum]
|
146
|
+
end
|
147
|
+
|
148
|
+
# Get the minimal duration of a specific category.
|
149
|
+
# <tt>cat</tt> The category
|
150
|
+
def min(cat)
|
151
|
+
@categories[cat][:min]
|
152
|
+
end
|
153
|
+
|
154
|
+
# Get the maximum duration of a specific category.
|
155
|
+
# <tt>cat</tt> The category
|
156
|
+
def max(cat)
|
157
|
+
@categories[cat][:max]
|
158
|
+
end
|
159
|
+
|
160
|
+
# Get the average duration of a specific category.
|
161
|
+
# <tt>cat</tt> The category
|
162
|
+
def mean(cat)
|
163
|
+
@categories[cat][:mean]
|
164
|
+
end
|
165
|
+
|
166
|
+
# Get the standard deviation of the duration of a specific category.
|
167
|
+
# <tt>cat</tt> The category
|
168
|
+
def stddev(cat)
|
169
|
+
Math.sqrt(variance(cat))
|
170
|
+
end
|
171
|
+
|
172
|
+
# Get the variance of the duration of a specific category.
|
173
|
+
# <tt>cat</tt> The category
|
174
|
+
def variance(cat)
|
175
|
+
return 0.0 if @categories[cat][:hits] <= 1
|
176
|
+
(@categories[cat][:sum_of_squares] / (@categories[cat][:hits] - 1))
|
177
|
+
end
|
178
|
+
|
179
|
+
# Get the average duration of a all categories.
|
180
|
+
def mean_overall
|
181
|
+
sum_overall / hits_overall
|
182
|
+
end
|
183
|
+
|
184
|
+
# Get the cumlative duration of a all categories.
|
185
|
+
def sum_overall
|
186
|
+
@categories.inject(0.0) { |sum, (name, cat)| sum + cat[:sum] }
|
187
|
+
end
|
188
|
+
|
189
|
+
# Get the total hits of a all categories.
|
190
|
+
def hits_overall
|
191
|
+
@categories.inject(0) { |sum, (name, cat)| sum + cat[:hits] }
|
192
|
+
end
|
193
|
+
|
194
|
+
# Return categories sorted by a given key.
|
195
|
+
# <tt>by</tt> The key to sort on. This parameter can be omitted if a sorting block is provided instead
|
196
|
+
def sorted_by(by = nil)
|
197
|
+
if block_given?
|
198
|
+
categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }
|
199
|
+
else
|
200
|
+
categories.sort { |a, b| send(by, b[0]) <=> send(by, a[0]) }
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
# Returns the column header for a statistics table to report on the statistics result
|
205
|
+
def statistics_header(options)
|
206
|
+
[
|
207
|
+
{:title => options[:title], :width => :rest},
|
208
|
+
{:title => 'Hits', :align => :right, :highlight => (options[:highlight] == :hits), :min_width => 4},
|
209
|
+
{:title => 'Sum', :align => :right, :highlight => (options[:highlight] == :sum), :min_width => 6},
|
210
|
+
{:title => 'Mean', :align => :right, :highlight => (options[:highlight] == :mean), :min_width => 6},
|
211
|
+
{:title => 'StdDev', :align => :right, :highlight => (options[:highlight] == :stddev), :min_width => 6},
|
212
|
+
{:title => 'Min', :align => :right, :highlight => (options[:highlight] == :min), :min_width => 6},
|
213
|
+
{:title => 'Max', :align => :right, :highlight => (options[:highlight] == :max), :min_width => 6}
|
214
|
+
]
|
215
|
+
end
|
216
|
+
|
217
|
+
# Returns a row of statistics information for a report table, given a category
|
218
|
+
def statistics_row(cat)
|
219
|
+
[cat, hits(cat), display_value(sum(cat)), display_value(mean(cat)), display_value(stddev(cat)),
|
220
|
+
display_value(min(cat)), display_value(max(cat))]
|
221
|
+
end
|
222
|
+
end
|
223
|
+
end
|
@@ -9,55 +9,22 @@ module RequestLogAnalyzer::Tracker
|
|
9
9
|
# * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
|
10
10
|
# * <tt>:title</tt> Title do be displayed above the report
|
11
11
|
# * <tt>:unless</tt> Handle request if this proc is false for the handled request.
|
12
|
-
class Traffic <
|
13
|
-
|
14
|
-
attr_reader :categories
|
15
|
-
|
16
|
-
include RequestLogAnalyzer::Tracker::StatisticsTracking
|
12
|
+
class Traffic < NumericValue
|
17
13
|
|
18
14
|
# Check if duration and catagory option have been received,
|
19
15
|
def prepare
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
@categorizer = create_lambda(options[:category])
|
24
|
-
@trafficizer = create_lambda(options[:traffic])
|
25
|
-
@categories = {}
|
26
|
-
end
|
27
|
-
|
28
|
-
|
29
|
-
# Get the duration information from the request and store it in the different categories.
|
30
|
-
# <tt>request</tt> The request.
|
31
|
-
def update(request)
|
32
|
-
category = @categorizer.call(request)
|
33
|
-
traffic = @trafficizer.call(request)
|
34
|
-
update_statistics(category, traffic) if traffic.kind_of?(Numeric) && !category.nil?
|
35
|
-
end
|
36
|
-
|
37
|
-
# Block function to build a result table using a provided sorting function.
|
38
|
-
# <tt>output</tt> The output object.
|
39
|
-
# <tt>amount</tt> The number of rows in the report table (default 10).
|
40
|
-
# === Options
|
41
|
-
# * </tt>:title</tt> The title of the table
|
42
|
-
# * </tt>:sort</tt> The key to sort on (:hits, :cumulative, :average, :min or :max)
|
43
|
-
def report_table(output, sort, options = {}, &block)
|
44
|
-
output.puts
|
45
|
-
|
46
|
-
top_categories = output.slice_results(sorted_by(sort))
|
47
|
-
output.with_style(:top_line => true) do
|
48
|
-
output.table(*statistics_header(:title => options[:title],:highlight => sort)) do |rows|
|
49
|
-
top_categories.each { |(cat, info)| rows.push(statistics_row(cat)) }
|
50
|
-
end
|
51
|
-
end
|
52
|
-
output.puts
|
16
|
+
options[:value] = options[:traffic] if options[:traffic]
|
17
|
+
options[:total] = true
|
18
|
+
super
|
53
19
|
end
|
54
20
|
|
55
21
|
# Formats the traffic number using x B/kB/MB/GB etc notation
|
56
22
|
def display_value(bytes)
|
57
23
|
return "-" if bytes.nil?
|
58
24
|
return "0 B" if bytes.zero?
|
25
|
+
|
59
26
|
case Math.log10(bytes).floor
|
60
|
-
when
|
27
|
+
when 0...4 then '%d B' % bytes
|
61
28
|
when 4...7 then '%d kB' % (bytes / 1000)
|
62
29
|
when 7...10 then '%d MB' % (bytes / 1000_000)
|
63
30
|
when 10...13 then '%d GB' % (bytes / 1000_000_000)
|
@@ -65,42 +32,9 @@ module RequestLogAnalyzer::Tracker
|
|
65
32
|
end
|
66
33
|
end
|
67
34
|
|
68
|
-
# Generate a request duration report to the given output object
|
69
|
-
# By default colulative and average duration are generated.
|
70
|
-
# Any options for the report should have been set during initialize.
|
71
|
-
# <tt>output</tt> The output object
|
72
|
-
def report(output)
|
73
|
-
|
74
|
-
sortings = output.options[:sort] || [:sum, :mean]
|
75
|
-
|
76
|
-
sortings.each do |report|
|
77
|
-
case report
|
78
|
-
when :mean
|
79
|
-
report_table(output, :mean, :title => "#{title} - sorted by mean")
|
80
|
-
when :stddev
|
81
|
-
report_table(output, :stddev, :title => "#{title} - sorted by standard deviation")
|
82
|
-
when :sum
|
83
|
-
report_table(output, :sum, :title => "#{title} - sorted by sum")
|
84
|
-
when :hits
|
85
|
-
report_table(output, :hits, :title => "#{title} - sorted by hits")
|
86
|
-
else
|
87
|
-
raise "Unknown duration report specified: #{report}!"
|
88
|
-
end
|
89
|
-
end
|
90
|
-
|
91
|
-
output.puts
|
92
|
-
output.puts "#{output.colorize(title, :white, :bold)} - observed total: " + output.colorize(display_value(sum_overall), :brown, :bold)
|
93
|
-
end
|
94
|
-
|
95
35
|
# Returns the title of this tracker for reports
|
96
36
|
def title
|
97
37
|
options[:title] || 'Request traffic'
|
98
38
|
end
|
99
|
-
|
100
|
-
# Returns all the categories and the tracked duration as a hash than can be exported to YAML
|
101
|
-
def to_yaml_object
|
102
|
-
return nil if @categories.empty?
|
103
|
-
@categories
|
104
|
-
end
|
105
39
|
end
|
106
40
|
end
|