request-log-analyzer 1.3.7 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +3 -3
- data/README.rdoc +1 -1
- data/bin/request-log-analyzer +17 -14
- data/lib/cli/command_line_arguments.rb +51 -51
- data/lib/cli/database_console.rb +3 -3
- data/lib/cli/database_console_init.rb +2 -2
- data/lib/cli/progressbar.rb +10 -10
- data/lib/cli/tools.rb +3 -3
- data/lib/request_log_analyzer.rb +4 -4
- data/lib/request_log_analyzer/aggregator.rb +10 -10
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +9 -9
- data/lib/request_log_analyzer/aggregator/echo.rb +14 -9
- data/lib/request_log_analyzer/aggregator/summarizer.rb +26 -26
- data/lib/request_log_analyzer/controller.rb +153 -69
- data/lib/request_log_analyzer/database.rb +13 -13
- data/lib/request_log_analyzer/database/base.rb +17 -17
- data/lib/request_log_analyzer/database/connection.rb +3 -3
- data/lib/request_log_analyzer/database/request.rb +2 -2
- data/lib/request_log_analyzer/database/source.rb +1 -1
- data/lib/request_log_analyzer/file_format.rb +15 -15
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +16 -16
- data/lib/request_log_analyzer/file_format/apache.rb +20 -19
- data/lib/request_log_analyzer/file_format/merb.rb +12 -12
- data/lib/request_log_analyzer/file_format/rack.rb +4 -4
- data/lib/request_log_analyzer/file_format/rails.rb +146 -70
- data/lib/request_log_analyzer/file_format/rails_development.rb +4 -49
- data/lib/request_log_analyzer/filter.rb +6 -6
- data/lib/request_log_analyzer/filter/anonymize.rb +6 -6
- data/lib/request_log_analyzer/filter/field.rb +9 -9
- data/lib/request_log_analyzer/filter/timespan.rb +12 -10
- data/lib/request_log_analyzer/line_definition.rb +15 -14
- data/lib/request_log_analyzer/log_processor.rb +22 -22
- data/lib/request_log_analyzer/mailer.rb +15 -9
- data/lib/request_log_analyzer/output.rb +53 -12
- data/lib/request_log_analyzer/output/fixed_width.rb +40 -41
- data/lib/request_log_analyzer/output/html.rb +20 -20
- data/lib/request_log_analyzer/request.rb +35 -36
- data/lib/request_log_analyzer/source.rb +7 -7
- data/lib/request_log_analyzer/source/database_loader.rb +7 -7
- data/lib/request_log_analyzer/source/log_parser.rb +48 -43
- data/lib/request_log_analyzer/tracker.rb +128 -14
- data/lib/request_log_analyzer/tracker/duration.rb +39 -132
- data/lib/request_log_analyzer/tracker/frequency.rb +31 -32
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +20 -19
- data/lib/request_log_analyzer/tracker/timespan.rb +17 -17
- data/lib/request_log_analyzer/tracker/traffic.rb +36 -116
- data/request-log-analyzer.gemspec +19 -15
- data/spec/fixtures/rails_22.log +1 -1
- data/spec/integration/command_line_usage_spec.rb +1 -1
- data/spec/lib/helpers.rb +7 -7
- data/spec/lib/macros.rb +3 -3
- data/spec/lib/matchers.rb +41 -27
- data/spec/lib/mocks.rb +15 -14
- data/spec/lib/testing_format.rb +9 -9
- data/spec/spec_helper.rb +6 -6
- data/spec/unit/aggregator/database_inserter_spec.rb +13 -13
- data/spec/unit/aggregator/summarizer_spec.rb +4 -4
- data/spec/unit/controller/controller_spec.rb +2 -2
- data/spec/unit/controller/log_processor_spec.rb +1 -1
- data/spec/unit/database/base_class_spec.rb +19 -19
- data/spec/unit/database/connection_spec.rb +3 -3
- data/spec/unit/database/database_spec.rb +25 -25
- data/spec/unit/file_format/amazon_s3_format_spec.rb +5 -5
- data/spec/unit/file_format/apache_format_spec.rb +13 -13
- data/spec/unit/file_format/file_format_api_spec.rb +13 -13
- data/spec/unit/file_format/line_definition_spec.rb +24 -17
- data/spec/unit/file_format/merb_format_spec.rb +41 -45
- data/spec/unit/file_format/rails_format_spec.rb +157 -117
- data/spec/unit/filter/anonymize_filter_spec.rb +2 -2
- data/spec/unit/filter/field_filter_spec.rb +13 -13
- data/spec/unit/filter/filter_spec.rb +1 -1
- data/spec/unit/filter/timespan_filter_spec.rb +15 -15
- data/spec/unit/mailer_spec.rb +30 -0
- data/spec/unit/{source/request_spec.rb → request_spec.rb} +30 -30
- data/spec/unit/source/log_parser_spec.rb +27 -27
- data/spec/unit/tracker/duration_tracker_spec.rb +115 -78
- data/spec/unit/tracker/frequency_tracker_spec.rb +74 -63
- data/spec/unit/tracker/hourly_spread_spec.rb +28 -20
- data/spec/unit/tracker/timespan_tracker_spec.rb +25 -13
- data/spec/unit/tracker/tracker_api_spec.rb +13 -13
- data/spec/unit/tracker/traffic_tracker_spec.rb +81 -79
- data/tasks/github-gem.rake +125 -75
- data/tasks/request_log_analyzer.rake +2 -2
- metadata +8 -6
@@ -1,7 +1,7 @@
|
|
1
1
|
# The RequestLogAnalyzer::Source module contains all functionality that loads requests from a given source
|
2
2
|
# and feed them to the pipeline for further processing. The requests (see RequestLogAnalyzer::Request) that
|
3
|
-
# will be parsed from a source, will be piped throug filters (see RequestLogAnalyzer::Filter) and are then
|
4
|
-
# fed to an aggregator (see RequestLogAnalyzer::Aggregator). The source instance is thus the beginning of
|
3
|
+
# will be parsed from a source, will be piped throug filters (see RequestLogAnalyzer::Filter) and are then
|
4
|
+
# fed to an aggregator (see RequestLogAnalyzer::Aggregator). The source instance is thus the beginning of
|
5
5
|
# the RequestLogAnalyzer chain.
|
6
6
|
#
|
7
7
|
# - The base class for all sources is RequestLogAnalyzer::Source::Base. All source classes should inherit from this class.
|
@@ -14,13 +14,13 @@ module RequestLogAnalyzer::Source
|
|
14
14
|
def self.const_missing(const)
|
15
15
|
RequestLogAnalyzer::load_default_class_file(self, const)
|
16
16
|
end
|
17
|
-
|
17
|
+
|
18
18
|
# The base Source class. All other sources should inherit from this class.
|
19
19
|
#
|
20
20
|
# A source implememtation should at least implement the each_request method, which should yield
|
21
21
|
# RequestLogAnalyzer::Request instances that will be fed through the pipleine.
|
22
22
|
class Base
|
23
|
-
|
23
|
+
|
24
24
|
# A hash of options
|
25
25
|
attr_reader :options
|
26
26
|
|
@@ -49,14 +49,14 @@ module RequestLogAnalyzer::Source
|
|
49
49
|
@options = options
|
50
50
|
@file_format = format
|
51
51
|
end
|
52
|
-
|
52
|
+
|
53
53
|
# The prepare method is called before the RequestLogAnalyzer::Source::Base#each_request method is called.
|
54
54
|
# Use this method to implement any initialization that should occur before this source can produce Request
|
55
55
|
# instances.
|
56
56
|
def prepare
|
57
57
|
end
|
58
|
-
|
59
|
-
# This function is called to actually produce the requests that will be send into the pipeline.
|
58
|
+
|
59
|
+
# This function is called to actually produce the requests that will be send into the pipeline.
|
60
60
|
# The implementation should yield instances of RequestLogAnalyzer::Request.
|
61
61
|
# <tt>options</tt>:: A Hash of options that can be used in the implementation.
|
62
62
|
def each_request(options = {}, &block) # :yields: request
|
@@ -2,7 +2,7 @@ require 'rubygems'
|
|
2
2
|
require 'activerecord'
|
3
3
|
|
4
4
|
module RequestLogAnalyzer::Source
|
5
|
-
|
5
|
+
|
6
6
|
# Active Resource hook
|
7
7
|
class Request < ActiveRecord::Base
|
8
8
|
has_many :completed_lines
|
@@ -40,7 +40,7 @@ module RequestLogAnalyzer::Source
|
|
40
40
|
@parsed_requests = 0
|
41
41
|
@requests = []
|
42
42
|
end
|
43
|
-
|
43
|
+
|
44
44
|
# Reads the input, which can either be a file, sequence of files or STDIN to parse
|
45
45
|
# lines specified in the FileFormat. This lines will be combined into Request instances,
|
46
46
|
# that will be yielded. The actual parsing occurs in the parse_io method.
|
@@ -52,10 +52,10 @@ module RequestLogAnalyzer::Source
|
|
52
52
|
RequestLogAnalyzer::Source::Request.find(:all).each do |request|
|
53
53
|
@parsed_requests += 1
|
54
54
|
@progress_handler.call(:progress, @parsed_requests) if @progress_handler
|
55
|
-
|
55
|
+
|
56
56
|
yield request.convert(self.file_format)
|
57
57
|
end
|
58
|
-
|
58
|
+
|
59
59
|
@progress_handler.call(:finished, @source_files) if @progress_handler
|
60
60
|
end
|
61
61
|
|
@@ -66,15 +66,15 @@ module RequestLogAnalyzer::Source
|
|
66
66
|
end
|
67
67
|
|
68
68
|
# Add a block to this method to install a warning handler while parsing,
|
69
|
-
# <tt>proc</tt>:: The proc that will be called to handle parse warning messages
|
69
|
+
# <tt>proc</tt>:: The proc that will be called to handle parse warning messages
|
70
70
|
def warning=(proc)
|
71
71
|
@warning_handler = proc
|
72
72
|
end
|
73
73
|
|
74
74
|
# This method is called by the parser if it encounteres any parsing problems.
|
75
|
-
# It will call the installed warning handler if any.
|
75
|
+
# It will call the installed warning handler if any.
|
76
76
|
#
|
77
|
-
# By default, RequestLogAnalyzer::Controller will install a warning handler
|
77
|
+
# By default, RequestLogAnalyzer::Controller will install a warning handler
|
78
78
|
# that will pass the warnings to each aggregator so they can do something useful
|
79
79
|
# with it.
|
80
80
|
#
|
@@ -1,13 +1,13 @@
|
|
1
1
|
module RequestLogAnalyzer::Source
|
2
|
-
|
2
|
+
|
3
3
|
# The LogParser class reads log data from a given source and uses a file format definition
|
4
|
-
# to parse all relevent information about requests from the file. A FileFormat module should
|
4
|
+
# to parse all relevent information about requests from the file. A FileFormat module should
|
5
5
|
# be provided that contains the definitions of the lines that occur in the log data.
|
6
6
|
#
|
7
|
-
# De order in which lines occur is used to combine lines to a single request. If these lines
|
8
|
-
# are mixed, requests cannot be combined properly. This can be the case if data is written to
|
9
|
-
# the log file simultaneously by different mongrel processes. This problem is detected by the
|
10
|
-
# parser. It will emit warnings when this occurs. LogParser supports multiple parse strategies
|
7
|
+
# De order in which lines occur is used to combine lines to a single request. If these lines
|
8
|
+
# are mixed, requests cannot be combined properly. This can be the case if data is written to
|
9
|
+
# the log file simultaneously by different mongrel processes. This problem is detected by the
|
10
|
+
# parser. It will emit warnings when this occurs. LogParser supports multiple parse strategies
|
11
11
|
# that deal differently with this problem.
|
12
12
|
class LogParser < Base
|
13
13
|
|
@@ -15,7 +15,7 @@ module RequestLogAnalyzer::Source
|
|
15
15
|
|
16
16
|
# The default parse strategy that will be used to parse the input.
|
17
17
|
DEFAULT_PARSE_STRATEGY = 'assume-correct'
|
18
|
-
|
18
|
+
|
19
19
|
# All available parse strategies.
|
20
20
|
PARSE_STRATEGIES = ['cautious', 'assume-correct']
|
21
21
|
|
@@ -33,33 +33,38 @@ module RequestLogAnalyzer::Source
|
|
33
33
|
@parsed_requests = 0
|
34
34
|
@skipped_lines = 0
|
35
35
|
@skipped_requests = 0
|
36
|
+
@current_request = nil
|
37
|
+
@current_source = nil
|
36
38
|
@current_file = nil
|
37
39
|
@current_lineno = nil
|
38
40
|
@source_files = options[:source_files]
|
39
|
-
|
41
|
+
@progress_handler = nil
|
42
|
+
|
40
43
|
@options[:parse_strategy] ||= DEFAULT_PARSE_STRATEGY
|
41
44
|
raise "Unknown parse strategy" unless PARSE_STRATEGIES.include?(@options[:parse_strategy])
|
42
45
|
end
|
43
|
-
|
46
|
+
|
44
47
|
# Reads the input, which can either be a file, sequence of files or STDIN to parse
|
45
48
|
# lines specified in the FileFormat. This lines will be combined into Request instances,
|
46
49
|
# that will be yielded. The actual parsing occurs in the parse_io method.
|
47
50
|
# <tt>options</tt>:: A Hash of options that will be pased to parse_io.
|
48
51
|
def each_request(options = {}, &block) # :yields: :request, request
|
49
|
-
|
52
|
+
|
50
53
|
case @source_files
|
51
54
|
when IO
|
52
|
-
|
53
|
-
|
55
|
+
if @source_files == $stdin
|
56
|
+
puts "Parsing from the standard input. Press CTRL+C to finish." # FIXME: not here
|
57
|
+
end
|
58
|
+
parse_stream(@source_files, options, &block)
|
54
59
|
when String
|
55
|
-
parse_file(@source_files, options, &block)
|
60
|
+
parse_file(@source_files, options, &block)
|
56
61
|
when Array
|
57
|
-
parse_files(@source_files, options, &block)
|
62
|
+
parse_files(@source_files, options, &block)
|
58
63
|
else
|
59
64
|
raise "Unknown source provided"
|
60
65
|
end
|
61
66
|
end
|
62
|
-
|
67
|
+
|
63
68
|
# Make sure the Enumerable methods work as expected
|
64
69
|
alias_method :each, :each_request
|
65
70
|
|
@@ -70,12 +75,12 @@ module RequestLogAnalyzer::Source
|
|
70
75
|
def parse_files(files, options = {}, &block) # :yields: request
|
71
76
|
files.each { |file| parse_file(file, options, &block) }
|
72
77
|
end
|
73
|
-
|
78
|
+
|
74
79
|
# Check if a file has a compressed extention in the filename.
|
75
80
|
# If recognized, return the command string used to decompress the file
|
76
81
|
def decompress_file?(filename)
|
77
82
|
nice_command = "nice -n 5"
|
78
|
-
|
83
|
+
|
79
84
|
return "#{nice_command} gunzip -c -d #{filename}" if filename.match(/\.tar.gz$/) || filename.match(/\.tgz$/) || filename.match(/\.gz$/)
|
80
85
|
return "#{nice_command} bunzip2 -c -d #{filename}" if filename.match(/\.bz2$/)
|
81
86
|
return "#{nice_command} unzip -p #{filename}" if filename.match(/\.zip$/)
|
@@ -96,22 +101,22 @@ module RequestLogAnalyzer::Source
|
|
96
101
|
|
97
102
|
@current_source = File.expand_path(file)
|
98
103
|
@source_changes_handler.call(:started, @current_source) if @source_changes_handler
|
99
|
-
|
104
|
+
|
100
105
|
if decompress_file?(file).empty?
|
101
106
|
|
102
107
|
@progress_handler = @dormant_progress_handler
|
103
108
|
@progress_handler.call(:started, file) if @progress_handler
|
104
|
-
|
109
|
+
|
105
110
|
File.open(file, 'r') { |f| parse_io(f, options, &block) }
|
106
|
-
|
111
|
+
|
107
112
|
@progress_handler.call(:finished, file) if @progress_handler
|
108
113
|
@progress_handler = nil
|
109
114
|
else
|
110
115
|
IO.popen(decompress_file?(file), 'r') { |f| parse_io(f, options, &block) }
|
111
116
|
end
|
112
|
-
|
117
|
+
|
113
118
|
@source_changes_handler.call(:finished, @current_source) if @source_changes_handler
|
114
|
-
|
119
|
+
|
115
120
|
@current_source = nil
|
116
121
|
|
117
122
|
end
|
@@ -119,7 +124,7 @@ module RequestLogAnalyzer::Source
|
|
119
124
|
# Parses an IO stream. It will simply call parse_io. This function does not support progress updates
|
120
125
|
# because the length of a stream is not known.
|
121
126
|
# <tt>stream</tt>:: The IO stream that should be parsed.
|
122
|
-
# <tt>options</tt>:: A Hash of options that will be pased to parse_io.
|
127
|
+
# <tt>options</tt>:: A Hash of options that will be pased to parse_io.
|
123
128
|
def parse_stream(stream, options = {}, &block)
|
124
129
|
parse_io(stream, options, &block)
|
125
130
|
end
|
@@ -140,15 +145,15 @@ module RequestLogAnalyzer::Source
|
|
140
145
|
@current_lineno = 1
|
141
146
|
while line = io.gets
|
142
147
|
@progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno % 255 == 0
|
143
|
-
|
148
|
+
|
144
149
|
if request_data = file_format.parse_line(line) { |wt, message| warn(wt, message) }
|
145
150
|
@parsed_lines += 1
|
146
151
|
update_current_request(request_data.merge(:source => @current_source, :lineno => @current_lineno), &block)
|
147
152
|
end
|
148
|
-
|
153
|
+
|
149
154
|
@current_lineno += 1
|
150
155
|
end
|
151
|
-
|
156
|
+
|
152
157
|
warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
|
153
158
|
@current_lineno = nil
|
154
159
|
end
|
@@ -164,7 +169,7 @@ module RequestLogAnalyzer::Source
|
|
164
169
|
def warning=(proc)
|
165
170
|
@warning_handler = proc
|
166
171
|
end
|
167
|
-
|
172
|
+
|
168
173
|
# Add a block to this method to install a source change handler while parsing,
|
169
174
|
# <tt>proc</tt>:: The proc that will be called to handle source changes
|
170
175
|
def source_changes=(proc)
|
@@ -172,9 +177,9 @@ module RequestLogAnalyzer::Source
|
|
172
177
|
end
|
173
178
|
|
174
179
|
# This method is called by the parser if it encounteres any parsing problems.
|
175
|
-
# It will call the installed warning handler if any.
|
180
|
+
# It will call the installed warning handler if any.
|
176
181
|
#
|
177
|
-
# By default, RequestLogAnalyzer::Controller will install a warning handler
|
182
|
+
# By default, RequestLogAnalyzer::Controller will install a warning handler
|
178
183
|
# that will pass the warnings to each aggregator so they can do something useful
|
179
184
|
# with it.
|
180
185
|
#
|
@@ -186,24 +191,24 @@ module RequestLogAnalyzer::Source
|
|
186
191
|
|
187
192
|
protected
|
188
193
|
|
189
|
-
# Combines the different lines of a request into a single Request object. It will start a
|
190
|
-
# new request when a header line is encountered en will emit the request when a footer line
|
194
|
+
# Combines the different lines of a request into a single Request object. It will start a
|
195
|
+
# new request when a header line is encountered en will emit the request when a footer line
|
191
196
|
# is encountered.
|
192
197
|
#
|
193
198
|
# Combining the lines is done using heuristics. Problems can occur in this process. The
|
194
199
|
# current parse strategy defines how these cases are handled.
|
195
200
|
#
|
196
201
|
# When using the 'assume-correct' parse strategy (default):
|
197
|
-
# - Every line that is parsed before a header line is ignored as it cannot be included in
|
202
|
+
# - Every line that is parsed before a header line is ignored as it cannot be included in
|
198
203
|
# any request. It will emit a :no_current_request warning.
|
199
|
-
# - If a header line is found before the previous requests was closed, the previous request
|
204
|
+
# - If a header line is found before the previous requests was closed, the previous request
|
200
205
|
# will be yielded and a new request will be started.
|
201
206
|
#
|
202
207
|
# When using the 'cautious' parse strategy:
|
203
|
-
# - Every line that is parsed before a header line is ignored as it cannot be included in
|
208
|
+
# - Every line that is parsed before a header line is ignored as it cannot be included in
|
204
209
|
# any request. It will emit a :no_current_request warning.
|
205
210
|
# - A header line that is parsed before a request is closed by a footer line, is a sign of
|
206
|
-
# an unproperly ordered file. All data that is gathered for the request until then is
|
211
|
+
# an unproperly ordered file. All data that is gathered for the request until then is
|
207
212
|
# discarded and the next request is ignored as well. An :unclosed_request warning is
|
208
213
|
# emitted.
|
209
214
|
#
|
@@ -230,7 +235,7 @@ module RequestLogAnalyzer::Source
|
|
230
235
|
@current_request << request_data
|
231
236
|
if footer_line?(request_data)
|
232
237
|
handle_request(@current_request, &block) # yield @current_request
|
233
|
-
@current_request = nil
|
238
|
+
@current_request = nil
|
234
239
|
end
|
235
240
|
else
|
236
241
|
@skipped_lines += 1
|
@@ -238,8 +243,8 @@ module RequestLogAnalyzer::Source
|
|
238
243
|
end
|
239
244
|
end
|
240
245
|
end
|
241
|
-
|
242
|
-
# Handles the parsed request by sending it into the pipeline.
|
246
|
+
|
247
|
+
# Handles the parsed request by sending it into the pipeline.
|
243
248
|
#
|
244
249
|
# - It will call RequestLogAnalyzer::Request#validate on the request instance
|
245
250
|
# - It will send the request into the pipeline, checking whether it was accepted by all the filters.
|
@@ -251,19 +256,19 @@ module RequestLogAnalyzer::Source
|
|
251
256
|
request.validate
|
252
257
|
accepted = block_given? ? yield(request) : true
|
253
258
|
@skipped_requests += 1 unless accepted
|
254
|
-
end
|
259
|
+
end
|
255
260
|
|
256
261
|
# Checks whether a given line hash is a header line according to the current file format.
|
257
|
-
# <tt>hash</tt>:: A hash of data that was parsed from the line.
|
262
|
+
# <tt>hash</tt>:: A hash of data that was parsed from the line.
|
258
263
|
def header_line?(hash)
|
259
264
|
hash[:line_definition].header
|
260
265
|
end
|
261
266
|
|
262
|
-
# Checks whether a given line hash is a footer line according to the current file format.
|
263
|
-
# <tt>hash</tt>:: A hash of data that was parsed from the line.
|
267
|
+
# Checks whether a given line hash is a footer line according to the current file format.
|
268
|
+
# <tt>hash</tt>:: A hash of data that was parsed from the line.
|
264
269
|
def footer_line?(hash)
|
265
270
|
hash[:line_definition].footer
|
266
|
-
end
|
271
|
+
end
|
267
272
|
end
|
268
273
|
|
269
274
|
end
|
@@ -16,13 +16,13 @@ module RequestLogAnalyzer::Tracker
|
|
16
16
|
#
|
17
17
|
# For example :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
|
18
18
|
class Base
|
19
|
-
|
19
|
+
|
20
20
|
attr_reader :options
|
21
|
-
|
21
|
+
|
22
22
|
# Initialize the class
|
23
23
|
# Note that the options are only applicable if should_update? is not overwritten
|
24
24
|
# by the inheriting class.
|
25
|
-
#
|
25
|
+
#
|
26
26
|
# === Options
|
27
27
|
# * <tt>:if</tt> Handle request if this proc is true for the handled request.
|
28
28
|
# * <tt>:unless</tt> Handle request if this proc is false for the handled request.
|
@@ -31,7 +31,7 @@ module RequestLogAnalyzer::Tracker
|
|
31
31
|
@options = options
|
32
32
|
setup_should_update_checks!
|
33
33
|
end
|
34
|
-
|
34
|
+
|
35
35
|
# Sets up the tracker's should_update? checks.
|
36
36
|
def setup_should_update_checks!
|
37
37
|
@should_update_checks = []
|
@@ -42,24 +42,34 @@ module RequestLogAnalyzer::Tracker
|
|
42
42
|
@should_update_checks.push( lambda { |request| !request[options[:unless]] }) if options[:unless].kind_of?(Symbol)
|
43
43
|
end
|
44
44
|
|
45
|
+
# Creates a lambda expression to return a static field from a request. If the
|
46
|
+
# argument already is a lambda exprssion, it will simply return the argument.
|
47
|
+
def create_lambda(arg)
|
48
|
+
case arg
|
49
|
+
when Proc then arg
|
50
|
+
when Symbol then lambda { |request| request[arg] }
|
51
|
+
else raise "Canot create a lambda expression from this argument: #{arg.inspect}!"
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
45
55
|
# Hook things that need to be done before running here.
|
46
56
|
def prepare
|
47
57
|
end
|
48
|
-
|
58
|
+
|
49
59
|
# Will be called with each request.
|
50
60
|
# <tt>request</tt> The request to track data in.
|
51
61
|
def update(request)
|
52
62
|
end
|
53
|
-
|
63
|
+
|
54
64
|
# Hook things that need to be done after running here.
|
55
65
|
def finalize
|
56
66
|
end
|
57
|
-
|
67
|
+
|
58
68
|
# Determine if we should run the update function at all.
|
59
69
|
# Usually the update function will be heavy, so a light check is done here
|
60
70
|
# determining if we need to call update at all.
|
61
71
|
#
|
62
|
-
# Default this checks if defined:
|
72
|
+
# Default this checks if defined:
|
63
73
|
# * :line_type is also in the request hash.
|
64
74
|
# * :if is true for this request.
|
65
75
|
# * :unless if false for this request
|
@@ -68,25 +78,129 @@ module RequestLogAnalyzer::Tracker
|
|
68
78
|
def should_update?(request)
|
69
79
|
@should_update_checks.all? { |c| c.call(request) }
|
70
80
|
end
|
71
|
-
|
81
|
+
|
72
82
|
# Hook report generation here.
|
73
83
|
# Defaults to self.inspect
|
74
84
|
# <tt>output</tt> The output object the report will be passed to.
|
75
85
|
def report(output)
|
76
86
|
output << self.inspect
|
77
|
-
output << "\n"
|
87
|
+
output << "\n"
|
78
88
|
end
|
79
|
-
|
89
|
+
|
80
90
|
# The title of this tracker. Used for reporting.
|
81
91
|
def title
|
82
92
|
self.class.to_s
|
83
93
|
end
|
84
|
-
|
94
|
+
|
85
95
|
# This method is called by RequestLogAnalyzer::Aggregator:Summarizer to retrieve an
|
86
96
|
# object with all the results of this tracker, that can be dumped to YAML format.
|
87
97
|
def to_yaml_object
|
88
98
|
nil
|
89
99
|
end
|
90
|
-
|
91
|
-
|
100
|
+
end
|
101
|
+
|
102
|
+
module StatisticsTracking
|
103
|
+
|
104
|
+
# Update sthe running calculation of statistics with the newly found numeric value.
|
105
|
+
# <tt>category</tt>:: The category for which to update the running statistics calculations
|
106
|
+
# <tt>number</tt>:: The numeric value to update the calculations with.
|
107
|
+
def update_statistics(category, number)
|
108
|
+
@categories[category] ||= {:hits => 0, :sum => 0, :mean => 0.0, :sum_of_squares => 0.0, :min => number, :max => number }
|
109
|
+
delta = number - @categories[category][:mean]
|
110
|
+
|
111
|
+
@categories[category][:hits] += 1
|
112
|
+
@categories[category][:mean] += (delta / @categories[category][:hits])
|
113
|
+
@categories[category][:sum_of_squares] += delta * (number - @categories[category][:mean])
|
114
|
+
@categories[category][:sum] += number
|
115
|
+
@categories[category][:min] = number if number < @categories[category][:min]
|
116
|
+
@categories[category][:max] = number if number > @categories[category][:max]
|
117
|
+
end
|
118
|
+
|
119
|
+
# Get the number of hits of a specific category.
|
120
|
+
# <tt>cat</tt> The category
|
121
|
+
def hits(cat)
|
122
|
+
@categories[cat][:hits]
|
123
|
+
end
|
124
|
+
|
125
|
+
# Get the total duration of a specific category.
|
126
|
+
# <tt>cat</tt> The category
|
127
|
+
def sum(cat)
|
128
|
+
@categories[cat][:sum]
|
129
|
+
end
|
130
|
+
|
131
|
+
# Get the minimal duration of a specific category.
|
132
|
+
# <tt>cat</tt> The category
|
133
|
+
def min(cat)
|
134
|
+
@categories[cat][:min]
|
135
|
+
end
|
136
|
+
|
137
|
+
# Get the maximum duration of a specific category.
|
138
|
+
# <tt>cat</tt> The category
|
139
|
+
def max(cat)
|
140
|
+
@categories[cat][:max]
|
141
|
+
end
|
142
|
+
|
143
|
+
# Get the average duration of a specific category.
|
144
|
+
# <tt>cat</tt> The category
|
145
|
+
def mean(cat)
|
146
|
+
@categories[cat][:mean]
|
147
|
+
end
|
148
|
+
|
149
|
+
# Get the standard deviation of the duration of a specific category.
|
150
|
+
# <tt>cat</tt> The category
|
151
|
+
def stddev(cat)
|
152
|
+
Math.sqrt(variance(cat))
|
153
|
+
end
|
154
|
+
|
155
|
+
# Get the variance of the duration of a specific category.
|
156
|
+
# <tt>cat</tt> The category
|
157
|
+
def variance(cat)
|
158
|
+
return 0.0 if @categories[cat][:hits] <= 1
|
159
|
+
(@categories[cat][:sum_of_squares] / (@categories[cat][:hits] - 1))
|
160
|
+
end
|
161
|
+
|
162
|
+
# Get the average duration of a all categories.
|
163
|
+
def mean_overall
|
164
|
+
sum_overall / hits_overall
|
165
|
+
end
|
166
|
+
|
167
|
+
# Get the cumlative duration of a all categories.
|
168
|
+
def sum_overall
|
169
|
+
@categories.inject(0.0) { |sum, (name, cat)| sum + cat[:sum] }
|
170
|
+
end
|
171
|
+
|
172
|
+
# Get the total hits of a all categories.
|
173
|
+
def hits_overall
|
174
|
+
@categories.inject(0) { |sum, (name, cat)| sum + cat[:hits] }
|
175
|
+
end
|
176
|
+
|
177
|
+
# Return categories sorted by a given key.
|
178
|
+
# <tt>by</tt> The key to sort on. This parameter can be omitted if a sorting block is provided instead
|
179
|
+
def sorted_by(by = nil)
|
180
|
+
if block_given?
|
181
|
+
categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }
|
182
|
+
else
|
183
|
+
categories.sort { |a, b| send(by, b[0]) <=> send(by, a[0]) }
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
# Returns the column header for a statistics table to report on the statistics result
|
188
|
+
def statistics_header(options)
|
189
|
+
[
|
190
|
+
{:title => options[:title], :width => :rest},
|
191
|
+
{:title => 'Hits', :align => :right, :highlight => (options[:highlight] == :hits), :min_width => 4},
|
192
|
+
{:title => 'Sum', :align => :right, :highlight => (options[:highlight] == :sum), :min_width => 6},
|
193
|
+
{:title => 'Mean', :align => :right, :highlight => (options[:highlight] == :mean), :min_width => 6},
|
194
|
+
{:title => 'StdDev', :align => :right, :highlight => (options[:highlight] == :stddev), :min_width => 6},
|
195
|
+
{:title => 'Min', :align => :right, :highlight => (options[:highlight] == :min), :min_width => 6},
|
196
|
+
{:title => 'Max', :align => :right, :highlight => (options[:highlight] == :max), :min_width => 6}
|
197
|
+
]
|
198
|
+
end
|
199
|
+
|
200
|
+
# Returns a row of statistics information for a report table, given a category
|
201
|
+
def statistics_row(cat)
|
202
|
+
[cat, hits(cat), display_value(sum(cat)), display_value(mean(cat)), display_value(stddev(cat)),
|
203
|
+
display_value(min(cat)), display_value(max(cat))]
|
204
|
+
end
|
205
|
+
end
|
92
206
|
end
|