request-log-analyzer 1.3.4 → 1.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/cli/database_console_init.rb +2 -1
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +10 -8
- data/lib/request_log_analyzer/aggregator.rb +1 -5
- data/lib/request_log_analyzer/controller.rb +10 -21
- data/lib/request_log_analyzer/database/base.rb +7 -6
- data/lib/request_log_analyzer/database/request.rb +22 -0
- data/lib/request_log_analyzer/database/source.rb +13 -0
- data/lib/request_log_analyzer/database/warning.rb +14 -0
- data/lib/request_log_analyzer/database.rb +16 -91
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +6 -6
- data/lib/request_log_analyzer/file_format/apache.rb +21 -15
- data/lib/request_log_analyzer/file_format/merb.rb +21 -5
- data/lib/request_log_analyzer/file_format/rack.rb +11 -0
- data/lib/request_log_analyzer/file_format/rails.rb +8 -14
- data/lib/request_log_analyzer/file_format.rb +1 -13
- data/lib/request_log_analyzer/filter/anonymize.rb +2 -1
- data/lib/request_log_analyzer/filter.rb +6 -10
- data/lib/request_log_analyzer/log_processor.rb +6 -8
- data/lib/request_log_analyzer/output/fixed_width.rb +8 -7
- data/lib/request_log_analyzer/request.rb +61 -37
- data/lib/request_log_analyzer/source/database_loader.rb +3 -7
- data/lib/request_log_analyzer/source/log_parser.rb +3 -6
- data/lib/request_log_analyzer/source.rb +4 -6
- data/lib/request_log_analyzer/tracker/duration.rb +3 -3
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +1 -2
- data/lib/request_log_analyzer/tracker/traffic.rb +186 -0
- data/lib/request_log_analyzer/tracker.rb +12 -19
- data/lib/request_log_analyzer.rb +3 -3
- data/request-log-analyzer.gemspec +4 -4
- data/spec/database.yml +6 -0
- data/spec/lib/mocks.rb +5 -2
- data/spec/unit/aggregator/database_inserter_spec.rb +3 -3
- data/spec/unit/database/base_class_spec.rb +9 -16
- data/spec/unit/database/database_spec.rb +9 -14
- data/spec/unit/file_format/apache_format_spec.rb +38 -1
- data/spec/unit/tracker/tracker_api_spec.rb +111 -36
- data/spec/unit/tracker/traffic_tracker_spec.rb +105 -0
- metadata +13 -6
|
@@ -42,19 +42,7 @@ module RequestLogAnalyzer::FileFormat
|
|
|
42
42
|
raise "Invalid FileFormat class" unless klass.kind_of?(Class) && klass.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
|
|
43
43
|
|
|
44
44
|
@current_file_format = klass.create(*args) # return an instance of the class
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
# Makes classes aware of a file format by registering the file_format variable
|
|
48
|
-
module Awareness
|
|
49
|
-
|
|
50
|
-
def self.included(base)
|
|
51
|
-
base.send(:attr_reader, :file_format)
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
def register_file_format(format)
|
|
55
|
-
@file_format = format
|
|
56
|
-
end
|
|
57
|
-
end
|
|
45
|
+
end
|
|
58
46
|
|
|
59
47
|
# Base class for all log file format definitions. This class provides functions for subclasses to
|
|
60
48
|
# define their LineDefinitions and to define a summary report.
|
|
@@ -19,7 +19,8 @@ module RequestLogAnalyzer::Filter
|
|
|
19
19
|
value * ((75 + rand(50)) / 100.0)
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
-
def filter(request)
|
|
22
|
+
def filter(request)
|
|
23
|
+
# TODO: request.attributes is bad practice
|
|
23
24
|
request.attributes.each do |key, value|
|
|
24
25
|
if key == :ip
|
|
25
26
|
request.attributes[key] = generate_random_ip
|
|
@@ -7,28 +7,24 @@ module RequestLogAnalyzer::Filter
|
|
|
7
7
|
end
|
|
8
8
|
|
|
9
9
|
# Base filter class used to filter input requests.
|
|
10
|
-
# All filters should interit from this base.
|
|
10
|
+
# All filters should interit from this base.
|
|
11
11
|
class Base
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
attr_reader :log_parser
|
|
16
|
-
attr_reader :options
|
|
13
|
+
attr_reader :file_format, :options
|
|
17
14
|
|
|
18
15
|
# Initializer
|
|
19
16
|
# <tt>format</tt> The file format
|
|
20
17
|
# <tt>options</tt> Are passed to the filters.
|
|
21
18
|
def initialize(format, options = {})
|
|
22
|
-
@
|
|
23
|
-
|
|
19
|
+
@file_format = format
|
|
20
|
+
@options = options
|
|
24
21
|
end
|
|
25
22
|
|
|
26
23
|
# Return the request if the request should be kept.
|
|
27
24
|
# Return nil otherwise.
|
|
28
25
|
def filter(request)
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
end
|
|
26
|
+
request
|
|
27
|
+
end
|
|
32
28
|
end
|
|
33
29
|
|
|
34
30
|
end
|
|
@@ -12,11 +12,9 @@ module RequestLogAnalyzer
|
|
|
12
12
|
#
|
|
13
13
|
class LogProcessor
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
attr_reader :mode, :options, :sources
|
|
15
|
+
attr_reader :mode, :options, :sources, :file_format
|
|
18
16
|
attr_accessor :output_file
|
|
19
|
-
|
|
17
|
+
|
|
20
18
|
# Builds a logprocessor instance from the arguments given on the command line
|
|
21
19
|
# <tt>command</tt> The command hat was used to start the log processor. This will set the
|
|
22
20
|
# processing mode. Currently, only :strip is supported.
|
|
@@ -43,11 +41,11 @@ module RequestLogAnalyzer
|
|
|
43
41
|
# <tt>mode</tt> The processing mode
|
|
44
42
|
# <tt>options</tt> A hash with options to take into account
|
|
45
43
|
def initialize(format, mode, options = {})
|
|
46
|
-
@options
|
|
47
|
-
@mode
|
|
48
|
-
@sources
|
|
44
|
+
@options = options
|
|
45
|
+
@mode = mode
|
|
46
|
+
@sources = []
|
|
47
|
+
@file_format = format
|
|
49
48
|
$output_file = nil
|
|
50
|
-
self.register_file_format(format)
|
|
51
49
|
end
|
|
52
50
|
|
|
53
51
|
# Processes input files by opening it and sending the filestream to <code>process_io</code>,
|
|
@@ -104,7 +104,7 @@ module RequestLogAnalyzer::Output
|
|
|
104
104
|
end
|
|
105
105
|
|
|
106
106
|
# Write a link
|
|
107
|
-
# <tt>text</tt> The text in the link
|
|
107
|
+
# <tt>text</tt> The text in the link, or the URL itself if no text is given
|
|
108
108
|
# <tt>url</tt> The url to link to.
|
|
109
109
|
def link(text, url = nil)
|
|
110
110
|
if url.nil?
|
|
@@ -114,13 +114,13 @@ module RequestLogAnalyzer::Output
|
|
|
114
114
|
end
|
|
115
115
|
end
|
|
116
116
|
|
|
117
|
-
# Generate a header for a report
|
|
117
|
+
# Generate a header for a report
|
|
118
118
|
def header
|
|
119
119
|
if io.kind_of?(File)
|
|
120
|
-
puts "Request-log-analyzer summary report"
|
|
121
|
-
line
|
|
120
|
+
puts colorize("Request-log-analyzer summary report", :white, :bold)
|
|
121
|
+
line(:green)
|
|
122
122
|
puts "Version #{RequestLogAnalyzer::VERSION} - written by Willem van Bergen and Bart ten Brinke"
|
|
123
|
-
puts "
|
|
123
|
+
puts "Website: #{link('http://github.com/wvanbergen/request-log-analyzer')}"
|
|
124
124
|
end
|
|
125
125
|
end
|
|
126
126
|
|
|
@@ -128,8 +128,9 @@ module RequestLogAnalyzer::Output
|
|
|
128
128
|
def footer
|
|
129
129
|
puts
|
|
130
130
|
puts "Need an expert to analyze your application?"
|
|
131
|
-
puts "Mail to contact@railsdoctors.com or visit us at http://railsdoctors.com"
|
|
132
|
-
|
|
131
|
+
puts "Mail to #{link('contact@railsdoctors.com')} or visit us at #{link('http://railsdoctors.com')}."
|
|
132
|
+
line(:green)
|
|
133
|
+
puts "Thanks for using #{colorize('request-log-analyzer', :white, :bold)}!"
|
|
133
134
|
end
|
|
134
135
|
|
|
135
136
|
# Generate a report table and push it into the output object.
|
|
@@ -9,61 +9,80 @@ module RequestLogAnalyzer
|
|
|
9
9
|
# Request#every(field_name) returns all values corresponding to the given field name as array.
|
|
10
10
|
class Request
|
|
11
11
|
|
|
12
|
+
def self.inherited(klass)
|
|
13
|
+
# klass.send(:include, Converters)
|
|
14
|
+
end
|
|
15
|
+
|
|
12
16
|
module Converters
|
|
13
|
-
|
|
17
|
+
|
|
18
|
+
# Default converter function, which converts the parsed strings to a native Ruby type
|
|
19
|
+
# using the type indication in the line definition. It will use a custom connverter
|
|
20
|
+
# method if one is available.
|
|
14
21
|
def convert_value(value, capture_definition)
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
elsif !value.nil?
|
|
19
|
-
case capture_definition[:type]
|
|
20
|
-
when :decimal; value.to_f
|
|
21
|
-
when :float; value.to_f
|
|
22
|
-
when :double; value.to_f
|
|
23
|
-
when :integer; value.to_i
|
|
24
|
-
when :int; value.to_i
|
|
25
|
-
when :symbol; value.to_sym
|
|
26
|
-
else; value.to_s
|
|
27
|
-
end
|
|
28
|
-
else
|
|
29
|
-
nil
|
|
30
|
-
end
|
|
22
|
+
return capture_definition[:default] if value.nil?
|
|
23
|
+
custom_converter_method = :"convert_#{capture_definition[:type]}"
|
|
24
|
+
send(custom_converter_method, value, capture_definition)
|
|
31
25
|
end
|
|
32
26
|
|
|
27
|
+
def convert_string(value, capture_definition); value; end
|
|
28
|
+
def convert_decimal(value, capture_definition); value.to_f; end
|
|
29
|
+
def convert_float(value, capture_definition); value.to_f; end
|
|
30
|
+
def convert_decimal(value, capture_definition); value.to_f; end
|
|
31
|
+
def convert_int(value, capture_definition); value.to_i; end
|
|
32
|
+
def convert_integer(value, capture_definition); value.to_i; end
|
|
33
|
+
def convert_sym(value, capture_definition); value.to_sym; end
|
|
34
|
+
def convert_symbol(value, capture_definition); value.to_sym; end
|
|
35
|
+
|
|
36
|
+
# Converts :eval field, which should evaluate to a hash.
|
|
33
37
|
def convert_eval(value, capture_definition)
|
|
34
38
|
eval(value).inject({}) { |h, (k, v)| h[k.to_sym] = v; h}
|
|
35
39
|
rescue SyntaxError
|
|
36
40
|
nil
|
|
37
41
|
end
|
|
38
42
|
|
|
39
|
-
# Slow default method to parse timestamps
|
|
43
|
+
# Slow default method to parse timestamps.
|
|
44
|
+
# Reimplement this function in a file format specific Request class
|
|
45
|
+
# to improve the timestamp parsing speed.
|
|
40
46
|
def convert_timestamp(value, capture_definition)
|
|
41
|
-
DateTime.parse(value).strftime('%Y%m%d%H%M%S').to_i
|
|
47
|
+
DateTime.parse(value).strftime('%Y%m%d%H%M%S').to_i
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Converts traffic fields to (whole) bytes based on the given unit.
|
|
51
|
+
def convert_traffic(value, capture_definition)
|
|
52
|
+
case capture_definition[:unit]
|
|
53
|
+
when nil, :b, :B, :byte then value.to_i
|
|
54
|
+
when :GB, :G, :gigabyte then (value.to_f * 1000_000_000).round
|
|
55
|
+
when :GiB, :gibibyte then (value.to_f * (2 ** 30)).round
|
|
56
|
+
when :MB, :M, :megabyte then (value.to_f * 1000_000).round
|
|
57
|
+
when :MiB, :mebibyte then (value.to_f * (2 ** 20)).round
|
|
58
|
+
when :KB, :K, :kilobyte, :kB then (value.to_f * 1000).round
|
|
59
|
+
when :KiB, :kibibyte then (value.to_f * (2 ** 10)).round
|
|
60
|
+
else raise "Unknown traffic unit"
|
|
61
|
+
end
|
|
42
62
|
end
|
|
43
63
|
|
|
64
|
+
# Convert duration fields to float, and make sure the values are in seconds.
|
|
44
65
|
def convert_duration(value, capture_definition)
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
else
|
|
50
|
-
value.to_f
|
|
66
|
+
case capture_definition[:unit]
|
|
67
|
+
when nil, :sec, :s then value.to_f
|
|
68
|
+
when :microsec, :musec then value.to_f / 1000000.0
|
|
69
|
+
when :msec, :millisec then value.to_f / 1000.0
|
|
70
|
+
else raise "Unknown duration unit"
|
|
51
71
|
end
|
|
52
72
|
end
|
|
53
73
|
end
|
|
54
|
-
|
|
55
|
-
|
|
74
|
+
|
|
75
|
+
# Install the default converter methods
|
|
56
76
|
include Converters
|
|
57
|
-
|
|
58
|
-
attr_reader :lines
|
|
59
|
-
attr_reader :attributes
|
|
77
|
+
|
|
78
|
+
attr_reader :lines, :attributes, :file_format
|
|
60
79
|
|
|
61
80
|
# Initializes a new Request object.
|
|
62
81
|
# It will apply the the provided FileFormat module to this instance.
|
|
63
82
|
def initialize(file_format, attributes = {})
|
|
64
|
-
@lines
|
|
65
|
-
@attributes
|
|
66
|
-
|
|
83
|
+
@lines = []
|
|
84
|
+
@attributes = attributes
|
|
85
|
+
@file_format = file_format
|
|
67
86
|
end
|
|
68
87
|
|
|
69
88
|
# Creates a new request that was parsed from the log with the given FileFormat. The hashes
|
|
@@ -74,8 +93,10 @@ module RequestLogAnalyzer
|
|
|
74
93
|
return request
|
|
75
94
|
end
|
|
76
95
|
|
|
77
|
-
# Adds another line to the request.
|
|
78
|
-
#
|
|
96
|
+
# Adds another line to the request when it is parsed in the LogParser.
|
|
97
|
+
#
|
|
98
|
+
# The line should be provided as a hash with the attributes line_definition, :captures,
|
|
99
|
+
# :lineno and :source set. This function is called from LogParser.
|
|
79
100
|
def add_parsed_line (parsed_line)
|
|
80
101
|
value_hash = parsed_line[:line_definition].convert_captured_values(parsed_line[:captures], self)
|
|
81
102
|
value_hash[:line_type] = parsed_line[:line_definition].name
|
|
@@ -84,12 +105,16 @@ module RequestLogAnalyzer
|
|
|
84
105
|
add_line_hash(value_hash)
|
|
85
106
|
end
|
|
86
107
|
|
|
108
|
+
# Adds another line to the request using a plain hash.
|
|
109
|
+
#
|
|
110
|
+
# The line should be provides as a hash of the fields parsed from the line.
|
|
87
111
|
def add_line_hash(value_hash)
|
|
88
112
|
@lines << value_hash
|
|
89
113
|
@attributes = value_hash.merge(@attributes)
|
|
90
114
|
end
|
|
91
115
|
|
|
92
|
-
|
|
116
|
+
# Adds another line to the request. This method switches automatically between
|
|
117
|
+
# the add_line_hash and add_parsed_line based on the keys of the provided hash.
|
|
93
118
|
def <<(hash)
|
|
94
119
|
hash[:line_definition] ? add_parsed_line(hash) : add_line_hash(hash)
|
|
95
120
|
end
|
|
@@ -97,7 +122,6 @@ module RequestLogAnalyzer
|
|
|
97
122
|
# Checks whether the given line type was parsed from the log file for this request
|
|
98
123
|
def has_line_type?(line_type)
|
|
99
124
|
return true if @lines.length == 1 && @lines[0][:line_type] == line_type.to_sym
|
|
100
|
-
|
|
101
125
|
@lines.detect { |l| l[:line_type] == line_type.to_sym }
|
|
102
126
|
end
|
|
103
127
|
|
|
@@ -26,8 +26,7 @@ module RequestLogAnalyzer::Source
|
|
|
26
26
|
# The Database class gets log data from the database.
|
|
27
27
|
class DatabaseLoader < Base
|
|
28
28
|
|
|
29
|
-
attr_reader :source_files
|
|
30
|
-
attr_reader :requests
|
|
29
|
+
attr_reader :source_files, :file_format, :requests
|
|
31
30
|
|
|
32
31
|
# Initializes the log file parser instance.
|
|
33
32
|
# It will apply the language specific FileFormat module to this instance. It will use the line
|
|
@@ -35,14 +34,11 @@ module RequestLogAnalyzer::Source
|
|
|
35
34
|
#
|
|
36
35
|
# <tt>format</tt>:: The current file format instance
|
|
37
36
|
# <tt>options</tt>:: A hash of options that are used by the parser
|
|
38
|
-
def initialize(format, options = {})
|
|
39
|
-
|
|
40
|
-
@options = options
|
|
37
|
+
def initialize(format, options = {})
|
|
38
|
+
super(format, options)
|
|
41
39
|
@source_files = options[:source_files]
|
|
42
40
|
@parsed_requests = 0
|
|
43
41
|
@requests = []
|
|
44
|
-
|
|
45
|
-
self.register_file_format(format)
|
|
46
42
|
end
|
|
47
43
|
|
|
48
44
|
# Reads the input, which can either be a file, sequence of files or STDIN to parse
|
|
@@ -27,9 +27,8 @@ module RequestLogAnalyzer::Source
|
|
|
27
27
|
#
|
|
28
28
|
# <tt>format</tt>:: The current file format instance
|
|
29
29
|
# <tt>options</tt>:: A hash of options that are used by the parser
|
|
30
|
-
def initialize(format, options = {})
|
|
31
|
-
|
|
32
|
-
@options = options
|
|
30
|
+
def initialize(format, options = {})
|
|
31
|
+
super(format, options)
|
|
33
32
|
@parsed_lines = 0
|
|
34
33
|
@parsed_requests = 0
|
|
35
34
|
@skipped_lines = 0
|
|
@@ -40,8 +39,6 @@ module RequestLogAnalyzer::Source
|
|
|
40
39
|
|
|
41
40
|
@options[:parse_strategy] ||= DEFAULT_PARSE_STRATEGY
|
|
42
41
|
raise "Unknown parse strategy" unless PARSE_STRATEGIES.include?(@options[:parse_strategy])
|
|
43
|
-
|
|
44
|
-
self.register_file_format(format)
|
|
45
42
|
end
|
|
46
43
|
|
|
47
44
|
# Reads the input, which can either be a file, sequence of files or STDIN to parse
|
|
@@ -137,7 +134,7 @@ module RequestLogAnalyzer::Source
|
|
|
137
134
|
def parse_io(io, options = {}, &block) # :yields: request
|
|
138
135
|
@current_lineno = 1
|
|
139
136
|
while line = io.gets
|
|
140
|
-
@progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno
|
|
137
|
+
@progress_handler.call(:progress, io.pos) if @progress_handler && (@current_lineno & 255 == 0)
|
|
141
138
|
|
|
142
139
|
if request_data = file_format.parse_line(line) { |wt, message| warn(wt, message) }
|
|
143
140
|
@parsed_lines += 1
|
|
@@ -21,9 +21,6 @@ module RequestLogAnalyzer::Source
|
|
|
21
21
|
# RequestLogAnalyzer::Request instances that will be fed through the pipleine.
|
|
22
22
|
class Base
|
|
23
23
|
|
|
24
|
-
# Make the Spurce instance aware of the current file format
|
|
25
|
-
include RequestLogAnalyzer::FileFormat::Awareness
|
|
26
|
-
|
|
27
24
|
# A hash of options
|
|
28
25
|
attr_reader :options
|
|
29
26
|
|
|
@@ -42,14 +39,15 @@ module RequestLogAnalyzer::Source
|
|
|
42
39
|
# The total number of skipped requests because of filters.
|
|
43
40
|
attr_reader :skipped_requests
|
|
44
41
|
|
|
45
|
-
|
|
42
|
+
# The FileFormat instance that describes the format of this source.
|
|
43
|
+
attr_reader :file_format
|
|
46
44
|
|
|
47
45
|
# Initializer, which will register the file format and save any options given as a hash.
|
|
48
46
|
# <tt>format</tt>:: The file format instance
|
|
49
47
|
# <tt>options</tt>:: A hash of options that can be used by a specific Source implementation
|
|
50
48
|
def initialize(format, options = {})
|
|
51
|
-
@options
|
|
52
|
-
|
|
49
|
+
@options = options
|
|
50
|
+
@file_format = format
|
|
53
51
|
end
|
|
54
52
|
|
|
55
53
|
# The prepare method is called before the RequestLogAnalyzer::Source::Base#each_request method is called.
|
|
@@ -51,10 +51,10 @@ module RequestLogAnalyzer::Tracker
|
|
|
51
51
|
raise "Capture mismatch for multiple values in a request"
|
|
52
52
|
end
|
|
53
53
|
else
|
|
54
|
-
category = @categorizer.call(request)
|
|
55
|
-
duration = @durationizer.call(request)
|
|
54
|
+
category = @categorizer.call(request)
|
|
55
|
+
duration = @durationizer.call(request)
|
|
56
56
|
|
|
57
|
-
if duration.kind_of?(
|
|
57
|
+
if duration.kind_of?(Numeric) && !category.nil?
|
|
58
58
|
@categories[category] ||= {:hits => 0, :cumulative => 0.0, :min => duration, :max => duration }
|
|
59
59
|
@categories[category][:hits] += 1
|
|
60
60
|
@categories[category][:cumulative] += duration
|
|
@@ -41,8 +41,7 @@ module RequestLogAnalyzer::Tracker
|
|
|
41
41
|
# Check if the timestamp in the request and store it.
|
|
42
42
|
# <tt>request</tt> The request.
|
|
43
43
|
def update(request)
|
|
44
|
-
|
|
45
|
-
timestamp = request[options[:field]]
|
|
44
|
+
timestamp = request.first(options[:field])
|
|
46
45
|
|
|
47
46
|
@request_time_graph[timestamp.to_s[8..9].to_i] +=1
|
|
48
47
|
@first = timestamp if @first.nil? || timestamp < @first
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
module RequestLogAnalyzer::Tracker
|
|
2
|
+
|
|
3
|
+
# Analyze the average and total traffic of requests
|
|
4
|
+
#
|
|
5
|
+
# === Options
|
|
6
|
+
# * <tt>:amount</tt> The amount of lines in the report
|
|
7
|
+
# * <tt>:category</tt> Proc that handles request categorization for given fileformat (REQUEST_CATEGORIZER)
|
|
8
|
+
# * <tt>:traffic</tt> The field containing the duration in the request hash.
|
|
9
|
+
# * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
|
|
10
|
+
# * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
|
|
11
|
+
# * <tt>:title</tt> Title do be displayed above the report
|
|
12
|
+
# * <tt>:unless</tt> Handle request if this proc is false for the handled request.
|
|
13
|
+
class Traffic < Base
|
|
14
|
+
|
|
15
|
+
attr_reader :categories
|
|
16
|
+
|
|
17
|
+
# Check if duration and catagory option have been received,
|
|
18
|
+
def prepare
|
|
19
|
+
raise "No traffic field set up for category tracker #{self.inspect}" unless options[:traffic]
|
|
20
|
+
raise "No categorizer set up for duration tracker #{self.inspect}" unless options[:category]
|
|
21
|
+
|
|
22
|
+
@categorizer = options[:category].respond_to?(:call) ? options[:category] : lambda { |request| request[options[:category]] }
|
|
23
|
+
@trafficizer = options[:traffic].respond_to?(:call) ? options[:traffic] : lambda { |request| request[options[:traffic]] }
|
|
24
|
+
@categories = {}
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Get the duration information fron the request and store it in the different categories.
|
|
28
|
+
# <tt>request</tt> The request.
|
|
29
|
+
def update(request)
|
|
30
|
+
category = @categorizer.call(request)
|
|
31
|
+
traffic = @trafficizer.call(request)
|
|
32
|
+
|
|
33
|
+
if traffic.kind_of?(Numeric) && !category.nil?
|
|
34
|
+
@categories[category] ||= {:hits => 0, :cumulative => 0, :min => traffic, :max => traffic }
|
|
35
|
+
@categories[category][:hits] += 1
|
|
36
|
+
@categories[category][:cumulative] += traffic
|
|
37
|
+
@categories[category][:min] = traffic if traffic < @categories[category][:min]
|
|
38
|
+
@categories[category][:max] = traffic if traffic > @categories[category][:max]
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Get the number of hits of a specific category.
|
|
43
|
+
# <tt>cat</tt> The category
|
|
44
|
+
def hits(cat)
|
|
45
|
+
categories[cat][:hits]
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Get the total duration of a specific category.
|
|
49
|
+
# <tt>cat</tt> The category
|
|
50
|
+
def cumulative_traffic(cat)
|
|
51
|
+
categories[cat][:cumulative]
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Get the minimal duration of a specific category.
|
|
55
|
+
# <tt>cat</tt> The category
|
|
56
|
+
def min_traffic(cat)
|
|
57
|
+
categories[cat][:min]
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Get the maximum duration of a specific category.
|
|
61
|
+
# <tt>cat</tt> The category
|
|
62
|
+
def max_traffic(cat)
|
|
63
|
+
categories[cat][:max]
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Get the average duration of a specific category.
|
|
67
|
+
# <tt>cat</tt> The category
|
|
68
|
+
def average_traffic(cat)
|
|
69
|
+
categories[cat][:cumulative].to_f / categories[cat][:hits]
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Get the average duration of a all categories.
|
|
73
|
+
def overall_average_traffic
|
|
74
|
+
overall_cumulative_duration.to_f / overall_hits
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Get the cumlative duration of a all categories.
|
|
78
|
+
def overall_cumulative_traffic
|
|
79
|
+
categories.inject(0) { |sum, (name, cat)| sum + cat[:cumulative] }
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Get the total hits of a all categories.
|
|
83
|
+
def overall_hits
|
|
84
|
+
categories.inject(0) { |sum, (name, cat)| sum + cat[:hits] }
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Return categories sorted by hits.
|
|
88
|
+
def sorted_by_hits
|
|
89
|
+
sorted_by(:hits)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Return categories sorted by cumulative duration.
|
|
93
|
+
def sorted_by_cumulative
|
|
94
|
+
sorted_by(:cumulative)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Return categories sorted by cumulative duration.
|
|
98
|
+
def sorted_by_average
|
|
99
|
+
sorted_by { |cat| cat[:cumulative].to_f / cat[:hits] }
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Return categories sorted by a given key.
|
|
103
|
+
# <tt>by</tt> The key.
|
|
104
|
+
def sorted_by(by = nil)
|
|
105
|
+
if block_given?
|
|
106
|
+
categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }
|
|
107
|
+
else
|
|
108
|
+
categories.sort { |a, b| b[1][by] <=> a[1][by] }
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Block function to build a result table using a provided sorting function.
|
|
113
|
+
# <tt>output</tt> The output object.
|
|
114
|
+
# <tt>amount</tt> The number of rows in the report table (default 10).
|
|
115
|
+
# === Options
|
|
116
|
+
# * </tt>:title</tt> The title of the table
|
|
117
|
+
# * </tt>:sort</tt> The key to sort on (:hits, :cumulative, :average, :min or :max)
|
|
118
|
+
def report_table(output, amount = 10, options = {}, &block)
|
|
119
|
+
|
|
120
|
+
output.title(options[:title])
|
|
121
|
+
|
|
122
|
+
top_categories = @categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }.slice(0...amount)
|
|
123
|
+
output.table({:title => 'Category', :width => :rest},
|
|
124
|
+
{:title => 'Hits', :align => :right, :highlight => (options[:sort] == :hits), :min_width => 4},
|
|
125
|
+
{:title => 'Cumulative', :align => :right, :highlight => (options[:sort] == :cumulative), :min_width => 10},
|
|
126
|
+
{:title => 'Average', :align => :right, :highlight => (options[:sort] == :average), :min_width => 8},
|
|
127
|
+
{:title => 'Min', :align => :right, :highlight => (options[:sort] == :min)},
|
|
128
|
+
{:title => 'Max', :align => :right, :highlight => (options[:sort] == :max)}) do |rows|
|
|
129
|
+
|
|
130
|
+
top_categories.each do |(cat, info)|
|
|
131
|
+
rows << [cat, info[:hits], format_traffic(info[:cumulative]), format_traffic((info[:cumulative] / info[:hits]).round),
|
|
132
|
+
format_traffic(info[:min]), format_traffic(info[:max])]
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Formats the traffic number using x B/kB/MB/GB etc notation
|
|
138
|
+
def format_traffic(bytes)
|
|
139
|
+
return "0 B" if bytes.zero?
|
|
140
|
+
case Math.log10(bytes).floor
|
|
141
|
+
when 1...4 then '%d B' % bytes
|
|
142
|
+
when 4...7 then '%d kB' % (bytes / 1000)
|
|
143
|
+
when 7...10 then '%d MB' % (bytes / 1000_000)
|
|
144
|
+
when 10...13 then '%d GB' % (bytes / 1000_000_000)
|
|
145
|
+
else '%d TB' % (bytes / 1000_000_000_000)
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# Generate a request duration report to the given output object
|
|
150
|
+
# By default colulative and average duration are generated.
|
|
151
|
+
# Any options for the report should have been set during initialize.
|
|
152
|
+
# <tt>output</tt> The output object
|
|
153
|
+
def report(output)
|
|
154
|
+
|
|
155
|
+
options[:report] ||= [:cumulative, :average]
|
|
156
|
+
options[:top] ||= 20
|
|
157
|
+
|
|
158
|
+
options[:report].each do |report|
|
|
159
|
+
case report
|
|
160
|
+
when :average
|
|
161
|
+
report_table(output, options[:top], :title => "#{title} - top #{options[:top]} by average", :sort => :average) { |cat| cat[:cumulative] / cat[:hits] }
|
|
162
|
+
when :cumulative
|
|
163
|
+
report_table(output, options[:top], :title => "#{title} - top #{options[:top]} by sum", :sort => :cumulative) { |cat| cat[:cumulative] }
|
|
164
|
+
when :hits
|
|
165
|
+
report_table(output, options[:top], :title => "#{title} - top #{options[:top]} by hits", :sort => :hits) { |cat| cat[:hits] }
|
|
166
|
+
else
|
|
167
|
+
raise "Unknown duration report specified: #{report}!"
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
output.puts
|
|
172
|
+
output.puts "#{output.colorize(title, :white, :bold)} - observed total: " + output.colorize(format_traffic(overall_cumulative_traffic), :brown, :bold)
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# Returns the title of this tracker for reports
|
|
176
|
+
def title
|
|
177
|
+
options[:title] || 'Request traffic'
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# Returns all the categories and the tracked duration as a hash than can be exported to YAML
|
|
181
|
+
def to_yaml_object
|
|
182
|
+
return nil if @categories.empty?
|
|
183
|
+
@categories
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|
|
@@ -29,6 +29,17 @@ module RequestLogAnalyzer::Tracker
|
|
|
29
29
|
# * <tt>:line_type</tt> Line type this tracker will accept.
|
|
30
30
|
def initialize(options ={})
|
|
31
31
|
@options = options
|
|
32
|
+
setup_should_update_checks!
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Sets up the tracker's should_update? checks.
|
|
36
|
+
def setup_should_update_checks!
|
|
37
|
+
@should_update_checks = []
|
|
38
|
+
@should_update_checks.push( lambda { |request| request.has_line_type?(options[:line_type]) } ) if options[:line_type]
|
|
39
|
+
@should_update_checks.push(options[:if]) if options[:if].respond_to?(:call)
|
|
40
|
+
@should_update_checks.push( lambda { |request| request[options[:if]] }) if options[:if].kind_of?(Symbol)
|
|
41
|
+
@should_update_checks.push( lambda { |request| !options[:unless].call(request) }) if options[:unless].respond_to?(:call)
|
|
42
|
+
@should_update_checks.push( lambda { |request| !request[options[:unless]] }) if options[:unless].kind_of?(Symbol)
|
|
32
43
|
end
|
|
33
44
|
|
|
34
45
|
# Hook things that need to be done before running here.
|
|
@@ -55,25 +66,7 @@ module RequestLogAnalyzer::Tracker
|
|
|
55
66
|
#
|
|
56
67
|
# <tt>request</tt> The request object.
|
|
57
68
|
def should_update?(request)
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
if options[:if]
|
|
61
|
-
if options[:if].kind_of?(Symbol)
|
|
62
|
-
return false unless request[options[:if]]
|
|
63
|
-
elsif options[:if].respond_to?(:call)
|
|
64
|
-
return false unless options[:if].call(request)
|
|
65
|
-
end
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
if options[:unless]
|
|
69
|
-
if options[:unless].kind_of?(Symbol)
|
|
70
|
-
return false if request[options[:unless]]
|
|
71
|
-
elsif options[:unless].respond_to?(:call)
|
|
72
|
-
return false if options[:unless].call(request)
|
|
73
|
-
end
|
|
74
|
-
end
|
|
75
|
-
|
|
76
|
-
return true
|
|
69
|
+
@should_update_checks.all? { |c| c.call(request) }
|
|
77
70
|
end
|
|
78
71
|
|
|
79
72
|
# Hook report generation here.
|
data/lib/request_log_analyzer.rb
CHANGED
|
@@ -10,8 +10,8 @@ Encoding.default_external = 'binary' if defined? Encoding and Encoding.respond_t
|
|
|
10
10
|
module RequestLogAnalyzer
|
|
11
11
|
|
|
12
12
|
# The current version of request-log-analyzer.
|
|
13
|
-
# This will be diplayed in output reports etc.
|
|
14
|
-
VERSION = "1.3.
|
|
13
|
+
# This will be diplayed in output reports etc.
|
|
14
|
+
VERSION = "1.3.6"
|
|
15
15
|
|
|
16
16
|
# Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
|
|
17
17
|
# <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.
|
|
@@ -25,7 +25,7 @@ module RequestLogAnalyzer
|
|
|
25
25
|
# <tt>const</tt>:: The constant to load from the base constant as a string or symbol. This should be 'Bar' or :Bar when the constant Foo::Bar is being loaded.
|
|
26
26
|
def self.load_default_class_file(base, const)
|
|
27
27
|
require "#{to_underscore("#{base.name}::#{const}")}"
|
|
28
|
-
base.const_get(const)
|
|
28
|
+
base.const_get(const) if base.const_defined?(const)
|
|
29
29
|
end
|
|
30
30
|
|
|
31
31
|
# Convert a string/symbol in camelcase (RequestLogAnalyzer::Controller) to underscores (request_log_analyzer/controller)
|