request-log-analyzer 1.13.1 → 1.13.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/bin/console +17 -0
- data/lib/cli/command_line_arguments.rb +29 -36
- data/lib/cli/database_console.rb +1 -3
- data/lib/cli/database_console_init.rb +11 -11
- data/lib/cli/progressbar.rb +30 -32
- data/lib/cli/tools.rb +20 -23
- data/lib/request_log_analyzer.rb +8 -8
- data/lib/request_log_analyzer/aggregator.rb +4 -7
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +10 -13
- data/lib/request_log_analyzer/aggregator/echo.rb +5 -7
- data/lib/request_log_analyzer/aggregator/summarizer.rb +15 -18
- data/lib/request_log_analyzer/class_level_inheritable_attributes.rb +23 -0
- data/lib/request_log_analyzer/controller.rb +36 -42
- data/lib/request_log_analyzer/database.rb +4 -6
- data/lib/request_log_analyzer/database/base.rb +39 -41
- data/lib/request_log_analyzer/database/connection.rb +8 -10
- data/lib/request_log_analyzer/database/request.rb +1 -3
- data/lib/request_log_analyzer/database/source.rb +0 -2
- data/lib/request_log_analyzer/database/warning.rb +4 -6
- data/lib/request_log_analyzer/file_format.rb +46 -49
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +15 -19
- data/lib/request_log_analyzer/file_format/apache.rb +42 -45
- data/lib/request_log_analyzer/file_format/delayed_job.rb +13 -15
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job3.rb +5 -8
- data/lib/request_log_analyzer/file_format/delayed_job4.rb +5 -8
- data/lib/request_log_analyzer/file_format/haproxy.rb +44 -48
- data/lib/request_log_analyzer/file_format/merb.rb +13 -17
- data/lib/request_log_analyzer/file_format/mysql.rb +21 -25
- data/lib/request_log_analyzer/file_format/nginx.rb +0 -2
- data/lib/request_log_analyzer/file_format/oink.rb +30 -31
- data/lib/request_log_analyzer/file_format/postgresql.rb +11 -15
- data/lib/request_log_analyzer/file_format/rack.rb +0 -2
- data/lib/request_log_analyzer/file_format/rails.rb +100 -104
- data/lib/request_log_analyzer/file_format/rails3.rb +19 -23
- data/lib/request_log_analyzer/file_format/rails_development.rb +0 -1
- data/lib/request_log_analyzer/file_format/w3c.rb +16 -18
- data/lib/request_log_analyzer/filter.rb +0 -2
- data/lib/request_log_analyzer/filter/anonymize.rb +4 -7
- data/lib/request_log_analyzer/filter/field.rb +3 -6
- data/lib/request_log_analyzer/filter/timespan.rb +2 -6
- data/lib/request_log_analyzer/line_definition.rb +16 -19
- data/lib/request_log_analyzer/log_processor.rb +10 -14
- data/lib/request_log_analyzer/mailer.rb +9 -12
- data/lib/request_log_analyzer/output.rb +12 -14
- data/lib/request_log_analyzer/output/fixed_width.rb +21 -28
- data/lib/request_log_analyzer/output/html.rb +11 -14
- data/lib/request_log_analyzer/request.rb +53 -33
- data/lib/request_log_analyzer/source.rb +2 -5
- data/lib/request_log_analyzer/source/log_parser.rb +9 -16
- data/lib/request_log_analyzer/tracker.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +4 -6
- data/lib/request_log_analyzer/tracker/frequency.rb +9 -11
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -11
- data/lib/request_log_analyzer/tracker/numeric_value.rb +40 -44
- data/lib/request_log_analyzer/tracker/timespan.rb +5 -8
- data/lib/request_log_analyzer/tracker/traffic.rb +8 -10
- data/lib/request_log_analyzer/version.rb +1 -1
- data/request-log-analyzer.gemspec +6 -6
- data/spec/integration/command_line_usage_spec.rb +33 -33
- data/spec/integration/mailer_spec.rb +181 -185
- data/spec/integration/munin_plugins_rails_spec.rb +20 -20
- data/spec/integration/scout_spec.rb +40 -41
- data/spec/lib/helpers.rb +8 -9
- data/spec/lib/macros.rb +2 -4
- data/spec/lib/matchers.rb +20 -25
- data/spec/lib/mocks.rb +10 -11
- data/spec/lib/testing_format.rb +8 -10
- data/spec/spec_helper.rb +5 -1
- data/spec/unit/aggregator/database_inserter_spec.rb +23 -23
- data/spec/unit/aggregator/summarizer_spec.rb +7 -7
- data/spec/unit/controller/controller_spec.rb +14 -14
- data/spec/unit/controller/log_processor_spec.rb +3 -3
- data/spec/unit/database/base_class_spec.rb +36 -37
- data/spec/unit/database/connection_spec.rb +10 -10
- data/spec/unit/database/database_spec.rb +11 -11
- data/spec/unit/file_format/amazon_s3_format_spec.rb +66 -62
- data/spec/unit/file_format/apache_format_spec.rb +57 -52
- data/spec/unit/file_format/common_regular_expressions_spec.rb +18 -21
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job2_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job3_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job4_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job_format_spec.rb +12 -12
- data/spec/unit/file_format/file_format_api_spec.rb +19 -19
- data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
- data/spec/unit/file_format/haproxy_format_spec.rb +53 -49
- data/spec/unit/file_format/inheritance_spec.rb +13 -0
- data/spec/unit/file_format/line_definition_spec.rb +35 -33
- data/spec/unit/file_format/merb_format_spec.rb +13 -11
- data/spec/unit/file_format/mysql_format_spec.rb +24 -24
- data/spec/unit/file_format/oink_format_spec.rb +29 -29
- data/spec/unit/file_format/postgresql_format_spec.rb +9 -9
- data/spec/unit/file_format/rack_format_spec.rb +36 -31
- data/spec/unit/file_format/rails3_format_spec.rb +46 -46
- data/spec/unit/file_format/rails_format_spec.rb +52 -53
- data/spec/unit/file_format/w3c_format_spec.rb +27 -24
- data/spec/unit/filter/anonymize_filter_spec.rb +7 -7
- data/spec/unit/filter/field_filter_spec.rb +26 -26
- data/spec/unit/filter/filter_spec.rb +4 -4
- data/spec/unit/filter/timespan_filter_spec.rb +22 -22
- data/spec/unit/mailer_spec.rb +21 -21
- data/spec/unit/request_spec.rb +29 -29
- data/spec/unit/source/log_parser_spec.rb +5 -5
- data/spec/unit/tracker/duration_tracker_spec.rb +23 -23
- data/spec/unit/tracker/frequency_tracker_spec.rb +29 -30
- data/spec/unit/tracker/hourly_spread_spec.rb +35 -35
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +71 -72
- data/spec/unit/tracker/timespan_tracker_spec.rb +31 -31
- data/spec/unit/tracker/tracker_api_spec.rb +43 -44
- data/spec/unit/tracker/traffic_tracker_spec.rb +7 -7
- metadata +38 -35
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
module RequestLogAnalyzer::Tracker
|
|
2
|
-
|
|
3
2
|
# Base Tracker class. All other trackers inherit from this class
|
|
4
3
|
#
|
|
5
4
|
# Accepts the following options:
|
|
@@ -10,7 +9,6 @@ module RequestLogAnalyzer::Tracker
|
|
|
10
9
|
#
|
|
11
10
|
# For example :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
|
|
12
11
|
class Base
|
|
13
|
-
|
|
14
12
|
attr_reader :options
|
|
15
13
|
|
|
16
14
|
# Initialize the class
|
|
@@ -21,7 +19,7 @@ module RequestLogAnalyzer::Tracker
|
|
|
21
19
|
# * <tt>:if</tt> Handle request if this proc is true for the handled request.
|
|
22
20
|
# * <tt>:unless</tt> Handle request if this proc is false for the handled request.
|
|
23
21
|
# * <tt>:line_type</tt> Line type this tracker will accept.
|
|
24
|
-
def initialize(options ={})
|
|
22
|
+
def initialize(options = {})
|
|
25
23
|
@options = options
|
|
26
24
|
setup_should_update_checks!
|
|
27
25
|
end
|
|
@@ -29,20 +27,20 @@ module RequestLogAnalyzer::Tracker
|
|
|
29
27
|
# Sets up the tracker's should_update? checks.
|
|
30
28
|
def setup_should_update_checks!
|
|
31
29
|
@should_update_checks = []
|
|
32
|
-
@should_update_checks.push(
|
|
30
|
+
@should_update_checks.push(lambda { |request| request.has_line_type?(options[:line_type]) }) if options[:line_type]
|
|
33
31
|
@should_update_checks.push(options[:if]) if options[:if].respond_to?(:call)
|
|
34
|
-
@should_update_checks.push(
|
|
35
|
-
@should_update_checks.push(
|
|
36
|
-
@should_update_checks.push(
|
|
32
|
+
@should_update_checks.push(lambda { |request| request[options[:if]] }) if options[:if].is_a?(Symbol)
|
|
33
|
+
@should_update_checks.push(lambda { |request| !options[:unless].call(request) }) if options[:unless].respond_to?(:call)
|
|
34
|
+
@should_update_checks.push(lambda { |request| !request[options[:unless]] }) if options[:unless].is_a?(Symbol)
|
|
37
35
|
end
|
|
38
|
-
|
|
39
|
-
# Creates a lambda expression to return a static field from a request. If the
|
|
36
|
+
|
|
37
|
+
# Creates a lambda expression to return a static field from a request. If the
|
|
40
38
|
# argument already is a lambda exprssion, it will simply return the argument.
|
|
41
39
|
def create_lambda(arg)
|
|
42
40
|
case arg
|
|
43
41
|
when Proc then arg
|
|
44
42
|
when Symbol then lambda { |request| request[arg] }
|
|
45
|
-
else
|
|
43
|
+
else fail "Canot create a lambda expression from this argument: #{arg.inspect}!"
|
|
46
44
|
end
|
|
47
45
|
end
|
|
48
46
|
|
|
@@ -52,7 +50,7 @@ module RequestLogAnalyzer::Tracker
|
|
|
52
50
|
|
|
53
51
|
# Will be called with each request.
|
|
54
52
|
# <tt>request</tt> The request to track data in.
|
|
55
|
-
def update(
|
|
53
|
+
def update(_request)
|
|
56
54
|
end
|
|
57
55
|
|
|
58
56
|
# Hook things that need to be done after running here.
|
|
@@ -77,7 +75,7 @@ module RequestLogAnalyzer::Tracker
|
|
|
77
75
|
# Defaults to self.inspect
|
|
78
76
|
# <tt>output</tt> The output object the report will be passed to.
|
|
79
77
|
def report(output)
|
|
80
|
-
output <<
|
|
78
|
+
output << inspect
|
|
81
79
|
output << "\n"
|
|
82
80
|
end
|
|
83
81
|
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
module RequestLogAnalyzer::Tracker
|
|
2
|
-
|
|
3
2
|
# Analyze the duration of a specific attribute
|
|
4
3
|
#
|
|
5
4
|
# === Options
|
|
@@ -20,7 +19,6 @@ module RequestLogAnalyzer::Tracker
|
|
|
20
19
|
# EmployeeController#index.html [GET] | 5802 | 1477.32s | 0.25s
|
|
21
20
|
# .............
|
|
22
21
|
class Duration < NumericValue
|
|
23
|
-
|
|
24
22
|
# Check if duration and catagory option have been received,
|
|
25
23
|
def prepare
|
|
26
24
|
options[:value] = options[:duration] if options[:duration]
|
|
@@ -38,10 +36,10 @@ module RequestLogAnalyzer::Tracker
|
|
|
38
36
|
def display_value(time)
|
|
39
37
|
case time
|
|
40
38
|
when nil then '-'
|
|
41
|
-
when 0...1 then
|
|
42
|
-
when 1...60 then
|
|
43
|
-
when 60...3600 then
|
|
44
|
-
else
|
|
39
|
+
when 0...1 then '%0ims' % (time * 1000)
|
|
40
|
+
when 1...60 then '%0.02fs' % time
|
|
41
|
+
when 60...3600 then '%dm%02ds' % [time / 60, (time % 60).round]
|
|
42
|
+
else '%dh%02dm%02ds' % [time / 3600, (time % 3600) / 60, (time % 60).round]
|
|
45
43
|
end
|
|
46
44
|
end
|
|
47
45
|
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
module RequestLogAnalyzer::Tracker
|
|
2
|
-
|
|
3
2
|
# Catagorize requests by frequency.
|
|
4
3
|
# Count and analyze requests for a specific attribute
|
|
5
4
|
#
|
|
@@ -21,19 +20,18 @@ module RequestLogAnalyzer::Tracker
|
|
|
21
20
|
# POST | 11662 hits (24.2%) |=========
|
|
22
21
|
# DELETE | 512 hits (1.1%) |
|
|
23
22
|
class Frequency < Base
|
|
24
|
-
|
|
25
23
|
attr_reader :categories
|
|
26
24
|
|
|
27
25
|
# Check if categories are set up
|
|
28
26
|
def prepare
|
|
29
27
|
options[:category] = options[:value] if options[:value] && !options[:category]
|
|
30
|
-
|
|
31
|
-
|
|
28
|
+
fail "No categorizer set up for category tracker #{inspect}" unless options[:category]
|
|
29
|
+
|
|
32
30
|
@categorizer = create_lambda(options[:category]) unless options[:multiple]
|
|
33
|
-
|
|
34
|
-
# Initialize the categories. Use the list of category names to
|
|
31
|
+
|
|
32
|
+
# Initialize the categories. Use the list of category names to
|
|
35
33
|
@categories = {}
|
|
36
|
-
options[:all_categories].each { |cat| @categories[cat] = 0 } if options[:all_categories].
|
|
34
|
+
options[:all_categories].each { |cat| @categories[cat] = 0 } if options[:all_categories].is_a?(Enumerable)
|
|
37
35
|
end
|
|
38
36
|
|
|
39
37
|
# Check HTTP method of a request and store that in the categories hash.
|
|
@@ -45,9 +43,9 @@ module RequestLogAnalyzer::Tracker
|
|
|
45
43
|
if cat || options[:nils]
|
|
46
44
|
@categories[cat] ||= 0
|
|
47
45
|
@categories[cat] += 1
|
|
48
|
-
end
|
|
46
|
+
end
|
|
49
47
|
end
|
|
50
|
-
|
|
48
|
+
|
|
51
49
|
else
|
|
52
50
|
cat = @categorizer.call(request)
|
|
53
51
|
if cat || options[:nils]
|
|
@@ -65,7 +63,7 @@ module RequestLogAnalyzer::Tracker
|
|
|
65
63
|
|
|
66
64
|
# Return the overall frequency
|
|
67
65
|
def overall_frequency
|
|
68
|
-
categories.
|
|
66
|
+
categories.reduce(0) { |carry, item| carry + item[1] }
|
|
69
67
|
end
|
|
70
68
|
|
|
71
69
|
# Return the methods sorted by frequency
|
|
@@ -85,7 +83,7 @@ module RequestLogAnalyzer::Tracker
|
|
|
85
83
|
sorted_categories = output.slice_results(sorted_by_frequency)
|
|
86
84
|
total_hits = overall_frequency
|
|
87
85
|
|
|
88
|
-
output.table({:
|
|
86
|
+
output.table({ align: :left }, { align: :right }, { align: :right }, { type: :ratio, width: :rest }) do |rows|
|
|
89
87
|
sorted_categories.each do |(cat, count)|
|
|
90
88
|
rows << [cat, "#{count} hits", '%0.1f%%' % ((count.to_f / total_hits.to_f) * 100.0), (count.to_f / total_hits.to_f)]
|
|
91
89
|
end
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
module RequestLogAnalyzer::Tracker
|
|
2
|
-
|
|
3
2
|
# Determines the average hourly spread of the parsed requests.
|
|
4
3
|
# This spread is shown in a graph form.
|
|
5
4
|
#
|
|
@@ -29,31 +28,29 @@ module RequestLogAnalyzer::Tracker
|
|
|
29
28
|
# 18:00 - 107 hits : ==
|
|
30
29
|
# ................
|
|
31
30
|
class HourlySpread < Base
|
|
32
|
-
|
|
33
31
|
attr_reader :hour_frequencies, :first, :last
|
|
34
32
|
|
|
35
33
|
# Check if timestamp field is set in the options and prepare the result time graph.
|
|
36
34
|
def prepare
|
|
37
35
|
options[:field] ||= :timestamp
|
|
38
36
|
@hour_frequencies = (0...24).map { 0 }
|
|
39
|
-
@first, @last =
|
|
37
|
+
@first, @last = 99_999_999_999_999, 0
|
|
40
38
|
end
|
|
41
39
|
|
|
42
40
|
# Check if the timestamp in the request and store it.
|
|
43
41
|
# <tt>request</tt> The request.
|
|
44
42
|
def update(request)
|
|
45
43
|
timestamp = request.first(options[:field])
|
|
46
|
-
@hour_frequencies[timestamp.to_s[8..9].to_i] +=1
|
|
44
|
+
@hour_frequencies[timestamp.to_s[8..9].to_i] += 1
|
|
47
45
|
@first = timestamp if timestamp < @first
|
|
48
46
|
@last = timestamp if timestamp > @last
|
|
49
47
|
end
|
|
50
48
|
|
|
51
49
|
# Total amount of requests tracked
|
|
52
50
|
def total_requests
|
|
53
|
-
@hour_frequencies.
|
|
51
|
+
@hour_frequencies.reduce(0) { |sum, value| sum + value }
|
|
54
52
|
end
|
|
55
53
|
|
|
56
|
-
|
|
57
54
|
# First timestamp encountered
|
|
58
55
|
def first_timestamp
|
|
59
56
|
DateTime.parse(@first.to_s, '%Y%m%d%H%M%S') rescue nil
|
|
@@ -68,7 +65,7 @@ module RequestLogAnalyzer::Tracker
|
|
|
68
65
|
def timespan
|
|
69
66
|
last_timestamp - first_timestamp
|
|
70
67
|
end
|
|
71
|
-
|
|
68
|
+
|
|
72
69
|
# Generate an hourly spread report to the given output object.
|
|
73
70
|
# Any options for the report should have been set during initialize.
|
|
74
71
|
# <tt>output</tt> The output object
|
|
@@ -81,25 +78,25 @@ module RequestLogAnalyzer::Tracker
|
|
|
81
78
|
end
|
|
82
79
|
|
|
83
80
|
days = [1, timespan].max
|
|
84
|
-
output.table({}, {:
|
|
81
|
+
output.table({}, { align: :right }, { type: :ratio, width: :rest, treshold: 0.15 }) do |rows|
|
|
85
82
|
@hour_frequencies.each_with_index do |requests, index|
|
|
86
83
|
ratio = requests.to_f / total_requests.to_f
|
|
87
84
|
requests_per_day = (requests / days).ceil
|
|
88
|
-
rows << ["#{index.to_s.rjust(3)}:00",
|
|
85
|
+
rows << ["#{index.to_s.rjust(3)}:00", '%d hits/day' % requests_per_day, ratio]
|
|
89
86
|
end
|
|
90
87
|
end
|
|
91
88
|
end
|
|
92
89
|
|
|
93
90
|
# Returns the title of this tracker for reports
|
|
94
91
|
def title
|
|
95
|
-
options[:title] ||
|
|
92
|
+
options[:title] || 'Request distribution per hour'
|
|
96
93
|
end
|
|
97
94
|
|
|
98
95
|
# Returns the found frequencies per hour as a hash for YAML exporting
|
|
99
96
|
def to_yaml_object
|
|
100
97
|
yaml_object = {}
|
|
101
98
|
@hour_frequencies.each_with_index do |freq, hour|
|
|
102
|
-
yaml_object["#{hour}:00 - #{hour+1}:00"] = freq
|
|
99
|
+
yaml_object["#{hour}:00 - #{hour + 1}:00"] = freq
|
|
103
100
|
end
|
|
104
101
|
yaml_object
|
|
105
102
|
end
|
|
@@ -1,22 +1,19 @@
|
|
|
1
1
|
module RequestLogAnalyzer::Tracker
|
|
2
|
-
|
|
3
2
|
class NumericValue < Base
|
|
4
|
-
|
|
5
3
|
attr_reader :categories
|
|
6
4
|
|
|
7
5
|
# Sets up the numeric value tracker. It will check whether the value and category
|
|
8
6
|
# options are set that are used to extract and categorize the values during
|
|
9
7
|
# parsing. Two lambda procedures are created for these tasks
|
|
10
8
|
def prepare
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
raise "No categorizer set up for numeric tracker #{self.inspect}" unless options[:category]
|
|
9
|
+
fail "No value field set up for numeric tracker #{inspect}" unless options[:value]
|
|
10
|
+
fail "No categorizer set up for numeric tracker #{inspect}" unless options[:category]
|
|
14
11
|
|
|
15
12
|
unless options[:multiple]
|
|
16
13
|
@categorizer = create_lambda(options[:category])
|
|
17
14
|
@valueizer = create_lambda(options[:value])
|
|
18
15
|
end
|
|
19
|
-
|
|
16
|
+
|
|
20
17
|
@number_of_buckets = options[:number_of_buckets] || 1000
|
|
21
18
|
@min_bucket_value = options[:min_bucket_value] ? options[:min_bucket_value].to_f : 0.000001
|
|
22
19
|
@max_bucket_value = options[:max_bucket_value] ? options[:max_bucket_value].to_f : 1_000_000_000
|
|
@@ -44,15 +41,15 @@ module RequestLogAnalyzer::Tracker
|
|
|
44
41
|
if options[:multiple]
|
|
45
42
|
found_categories = request.every(options[:category])
|
|
46
43
|
found_values = request.every(options[:value])
|
|
47
|
-
|
|
44
|
+
fail 'Capture mismatch for multiple values in a request' unless found_categories.length == found_values.length
|
|
48
45
|
|
|
49
46
|
found_categories.each_with_index do |cat, index|
|
|
50
|
-
update_statistics(cat, found_values[index]) if cat && found_values[index].
|
|
47
|
+
update_statistics(cat, found_values[index]) if cat && found_values[index].is_a?(Numeric)
|
|
51
48
|
end
|
|
52
49
|
else
|
|
53
50
|
category = @categorizer.call(request)
|
|
54
51
|
value = @valueizer.call(request)
|
|
55
|
-
update_statistics(category, value) if (value.
|
|
52
|
+
update_statistics(category, value) if (value.is_a?(Numeric) || value.is_a?(Array)) && category
|
|
56
53
|
end
|
|
57
54
|
end
|
|
58
55
|
|
|
@@ -62,11 +59,11 @@ module RequestLogAnalyzer::Tracker
|
|
|
62
59
|
# === Options
|
|
63
60
|
# * </tt>:title</tt> The title of the table
|
|
64
61
|
# * </tt>:sort</tt> The key to sort on (:hits, :cumulative, :average, :min or :max)
|
|
65
|
-
def report_table(output, sort, options = {}, &
|
|
62
|
+
def report_table(output, sort, options = {}, &_block)
|
|
66
63
|
output.puts
|
|
67
64
|
top_categories = output.slice_results(sorted_by(sort))
|
|
68
|
-
output.with_style(:
|
|
69
|
-
output.table(*statistics_header(:
|
|
65
|
+
output.with_style(top_line: true) do
|
|
66
|
+
output.table(*statistics_header(title: options[:title], highlight: sort)) do |rows|
|
|
70
67
|
top_categories.each { |(category, _)| rows << statistics_row(category) }
|
|
71
68
|
end
|
|
72
69
|
end
|
|
@@ -74,16 +71,16 @@ module RequestLogAnalyzer::Tracker
|
|
|
74
71
|
|
|
75
72
|
# Display a value
|
|
76
73
|
def display_value(value)
|
|
77
|
-
return
|
|
78
|
-
return
|
|
74
|
+
return '- ' if value.nil?
|
|
75
|
+
return '0 ' if value.zero?
|
|
79
76
|
|
|
80
77
|
case [Math.log10(value.abs).floor, 0].max
|
|
81
78
|
when 0...4 then '%d ' % value
|
|
82
79
|
when 4...7 then '%dk' % (value / 1000)
|
|
83
|
-
when 7...10 then '%dM' % (value /
|
|
84
|
-
when 10...13 then '%dG' % (value /
|
|
85
|
-
when 13...16 then '%dT' % (value /
|
|
86
|
-
else '%dP' % (value /
|
|
80
|
+
when 7...10 then '%dM' % (value / 1_000_000)
|
|
81
|
+
when 10...13 then '%dG' % (value / 1_000_000_000)
|
|
82
|
+
when 13...16 then '%dT' % (value / 1_000_000_000_000)
|
|
83
|
+
else '%dP' % (value / 1_000_000_000_000_000)
|
|
87
84
|
end
|
|
88
85
|
end
|
|
89
86
|
|
|
@@ -94,7 +91,7 @@ module RequestLogAnalyzer::Tracker
|
|
|
94
91
|
def report(output)
|
|
95
92
|
sortings = output.options[:sort] || [:sum, :mean]
|
|
96
93
|
sortings.each do |sorting|
|
|
97
|
-
report_table(output, sorting, :
|
|
94
|
+
report_table(output, sorting, title: "#{title} - by #{sorting}")
|
|
98
95
|
end
|
|
99
96
|
|
|
100
97
|
if options[:total]
|
|
@@ -109,9 +106,9 @@ module RequestLogAnalyzer::Tracker
|
|
|
109
106
|
if options[:title]
|
|
110
107
|
options[:title]
|
|
111
108
|
else
|
|
112
|
-
title_builder =
|
|
113
|
-
title_builder << "#{options[:value]} " if options[:value].
|
|
114
|
-
title_builder << (options[:category].
|
|
109
|
+
title_builder = ''
|
|
110
|
+
title_builder << "#{options[:value]} " if options[:value].is_a?(Symbol)
|
|
111
|
+
title_builder << (options[:category].is_a?(Symbol) ? "per #{options[:category]}" : 'per request')
|
|
115
112
|
title_builder
|
|
116
113
|
end
|
|
117
114
|
end
|
|
@@ -154,8 +151,8 @@ module RequestLogAnalyzer::Tracker
|
|
|
154
151
|
# Returns a single value representing a bucket.
|
|
155
152
|
def bucket_value(index, type = nil)
|
|
156
153
|
case type
|
|
157
|
-
when :begin, :start, :lower, :lower_bound
|
|
158
|
-
when :end, :finish, :upper, :upper_bound
|
|
154
|
+
when :begin, :start, :lower, :lower_bound then bucket_lower_bound(index)
|
|
155
|
+
when :end, :finish, :upper, :upper_bound then bucket_upper_bound(index)
|
|
159
156
|
else bucket_average_value(index)
|
|
160
157
|
end
|
|
161
158
|
end
|
|
@@ -198,8 +195,7 @@ module RequestLogAnalyzer::Tracker
|
|
|
198
195
|
def percentile(category, x, type = nil)
|
|
199
196
|
bucket_value(percentile_index(category, x, type == :upper), type)
|
|
200
197
|
end
|
|
201
|
-
|
|
202
|
-
|
|
198
|
+
|
|
203
199
|
def median(category)
|
|
204
200
|
percentile(category, 50, :average)
|
|
205
201
|
end
|
|
@@ -217,8 +213,8 @@ module RequestLogAnalyzer::Tracker
|
|
|
217
213
|
Range.new(bucket_lower_bound(lower), bucket_upper_bound(upper))
|
|
218
214
|
when Numeric
|
|
219
215
|
percentile_interval(category, Range.new((100 - x) / 2, (100 - (100 - x) / 2)))
|
|
220
|
-
else
|
|
221
|
-
|
|
216
|
+
else
|
|
217
|
+
fail 'What does it mean?'
|
|
222
218
|
end
|
|
223
219
|
end
|
|
224
220
|
|
|
@@ -226,11 +222,11 @@ module RequestLogAnalyzer::Tracker
|
|
|
226
222
|
# <tt>category</tt>:: The category for which to update the running statistics calculations
|
|
227
223
|
# <tt>number</tt>:: The numeric value to update the calculations with.
|
|
228
224
|
def update_statistics(category, number)
|
|
229
|
-
return number.map {|n| update_statistics(category, n)} if number.is_a?(Array)
|
|
225
|
+
return number.map { |n| update_statistics(category, n) } if number.is_a?(Array)
|
|
226
|
+
|
|
227
|
+
@categories[category] ||= { hits: 0, sum: 0, mean: 0.0, sum_of_squares: 0.0, min: number, max: number,
|
|
228
|
+
buckets: Array.new(@number_of_buckets, 0) }
|
|
230
229
|
|
|
231
|
-
@categories[category] ||= { :hits => 0, :sum => 0, :mean => 0.0, :sum_of_squares => 0.0, :min => number, :max => number,
|
|
232
|
-
:buckets => Array.new(@number_of_buckets, 0) }
|
|
233
|
-
|
|
234
230
|
delta = number - @categories[category][:mean]
|
|
235
231
|
|
|
236
232
|
@categories[category][:hits] += 1
|
|
@@ -293,12 +289,12 @@ module RequestLogAnalyzer::Tracker
|
|
|
293
289
|
|
|
294
290
|
# Get the cumlative duration of a all categories.
|
|
295
291
|
def sum_overall
|
|
296
|
-
@categories.
|
|
292
|
+
@categories.reduce(0.0) { |sum, (_, cat)| sum + cat[:sum] }
|
|
297
293
|
end
|
|
298
294
|
|
|
299
295
|
# Get the total hits of a all categories.
|
|
300
296
|
def hits_overall
|
|
301
|
-
@categories.
|
|
297
|
+
@categories.reduce(0) { |sum, (_, cat)| sum + cat[:hits] }
|
|
302
298
|
end
|
|
303
299
|
|
|
304
300
|
# Return categories sorted by a given key.
|
|
@@ -314,22 +310,22 @@ module RequestLogAnalyzer::Tracker
|
|
|
314
310
|
# Returns the column header for a statistics table to report on the statistics result
|
|
315
311
|
def statistics_header(options)
|
|
316
312
|
[
|
|
317
|
-
{:
|
|
318
|
-
{:
|
|
319
|
-
{:
|
|
320
|
-
{:
|
|
321
|
-
{:
|
|
322
|
-
{:
|
|
323
|
-
{:
|
|
324
|
-
{:
|
|
313
|
+
{ title: options[:title], width: :rest },
|
|
314
|
+
{ title: 'Hits', align: :right, highlight: (options[:highlight] == :hits), min_width: 4 },
|
|
315
|
+
{ title: 'Sum', align: :right, highlight: (options[:highlight] == :sum), min_width: 6 },
|
|
316
|
+
{ title: 'Mean', align: :right, highlight: (options[:highlight] == :mean), min_width: 6 },
|
|
317
|
+
{ title: 'StdDev', align: :right, highlight: (options[:highlight] == :stddev), min_width: 6 },
|
|
318
|
+
{ title: 'Min', align: :right, highlight: (options[:highlight] == :min), min_width: 6 },
|
|
319
|
+
{ title: 'Max', align: :right, highlight: (options[:highlight] == :max), min_width: 6 },
|
|
320
|
+
{ title: '95 %tile', align: :right, highlight: (options[:highlight] == :percentile_interval), min_width: 11 }
|
|
325
321
|
]
|
|
326
322
|
end
|
|
327
323
|
|
|
328
324
|
# Returns a row of statistics information for a report table, given a category
|
|
329
325
|
def statistics_row(cat)
|
|
330
326
|
[cat, hits(cat), display_value(sum(cat)), display_value(mean(cat)), display_value(stddev(cat)),
|
|
331
|
-
|
|
332
|
-
|
|
327
|
+
display_value(min(cat)), display_value(max(cat)),
|
|
328
|
+
display_value(percentile_interval(cat, 95).begin) + '-' + display_value(percentile_interval(cat, 95).end)]
|
|
333
329
|
end
|
|
334
330
|
end
|
|
335
331
|
end
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
module RequestLogAnalyzer::Tracker
|
|
2
|
-
|
|
3
2
|
# Determines the datetime of the first request and the last request
|
|
4
3
|
# Also determines the amount of days inbetween these.
|
|
5
4
|
#
|
|
@@ -18,13 +17,12 @@ module RequestLogAnalyzer::Tracker
|
|
|
18
17
|
# Last request: 2008-07-20 06:18:06
|
|
19
18
|
# Total time analyzed: 7 days
|
|
20
19
|
class Timespan < Base
|
|
21
|
-
|
|
22
20
|
attr_reader :first, :last
|
|
23
21
|
|
|
24
22
|
# Check if timestamp field is set in the options.
|
|
25
23
|
def prepare
|
|
26
24
|
options[:field] ||= :timestamp
|
|
27
|
-
@first, @last =
|
|
25
|
+
@first, @last = 99_999_999_999_999, 0
|
|
28
26
|
end
|
|
29
27
|
|
|
30
28
|
# Check if the timestamp in the request and store it.
|
|
@@ -56,9 +54,9 @@ module RequestLogAnalyzer::Tracker
|
|
|
56
54
|
def report(output)
|
|
57
55
|
output.title(options[:title]) if options[:title]
|
|
58
56
|
|
|
59
|
-
if @last > 0 && @first <
|
|
60
|
-
output.with_style(:
|
|
61
|
-
output.table({:
|
|
57
|
+
if @last > 0 && @first < 99_999_999_999_999
|
|
58
|
+
output.with_style(cell_separator: false) do
|
|
59
|
+
output.table({ width: 20 }, {}) do |rows|
|
|
62
60
|
rows << ['First request:', first_timestamp.strftime('%Y-%m-%d %H:%M:%I')]
|
|
63
61
|
rows << ['Last request:', last_timestamp.strftime('%Y-%m-%d %H:%M:%I')]
|
|
64
62
|
rows << ['Total time analyzed:', "#{timespan.ceil} days"]
|
|
@@ -74,8 +72,7 @@ module RequestLogAnalyzer::Tracker
|
|
|
74
72
|
|
|
75
73
|
# A hash that can be exported to YAML with the first and last timestamp encountered.
|
|
76
74
|
def to_yaml_object
|
|
77
|
-
{ :
|
|
75
|
+
{ first: first_timestamp, last: last_timestamp }
|
|
78
76
|
end
|
|
79
|
-
|
|
80
77
|
end
|
|
81
78
|
end
|