wvanbergen-request-log-analyzer 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/DESIGN +14 -0
- data/HACKING +7 -0
- data/README.textile +9 -98
- data/Rakefile +2 -2
- data/bin/request-log-analyzer +1 -1
- data/lib/cli/bashcolorizer.rb +60 -0
- data/lib/cli/command_line_arguments.rb +301 -0
- data/lib/cli/progressbar.rb +236 -0
- data/lib/request_log_analyzer/aggregator/base.rb +51 -0
- data/lib/request_log_analyzer/aggregator/database.rb +97 -0
- data/lib/request_log_analyzer/aggregator/echo.rb +25 -0
- data/lib/request_log_analyzer/aggregator/summarizer.rb +116 -0
- data/lib/request_log_analyzer/controller.rb +206 -0
- data/lib/request_log_analyzer/file_format/merb.rb +33 -0
- data/lib/request_log_analyzer/file_format/rails.rb +119 -0
- data/lib/request_log_analyzer/file_format.rb +77 -0
- data/lib/request_log_analyzer/filter/base.rb +29 -0
- data/lib/request_log_analyzer/filter/field.rb +36 -0
- data/lib/request_log_analyzer/filter/timespan.rb +32 -0
- data/lib/request_log_analyzer/line_definition.rb +159 -0
- data/lib/request_log_analyzer/log_parser.rb +183 -0
- data/lib/request_log_analyzer/log_processor.rb +121 -0
- data/lib/request_log_analyzer/request.rb +115 -0
- data/lib/request_log_analyzer/source/base.rb +42 -0
- data/lib/request_log_analyzer/source/log_file.rb +180 -0
- data/lib/request_log_analyzer/tracker/base.rb +54 -0
- data/lib/request_log_analyzer/tracker/category.rb +71 -0
- data/lib/request_log_analyzer/tracker/duration.rb +81 -0
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +80 -0
- data/lib/request_log_analyzer/tracker/timespan.rb +54 -0
- data/spec/file_format_spec.rb +78 -0
- data/spec/file_formats/spec_format.rb +26 -0
- data/spec/filter_spec.rb +137 -0
- data/spec/log_processor_spec.rb +57 -0
- data/tasks/rspec.rake +6 -0
- metadata +53 -55
- data/TODO +0 -58
- data/bin/request-log-database +0 -81
- data/lib/base/log_parser.rb +0 -78
- data/lib/base/record_inserter.rb +0 -139
- data/lib/command_line/arguments.rb +0 -129
- data/lib/command_line/flag.rb +0 -51
- data/lib/merb_analyzer/log_parser.rb +0 -26
- data/lib/rails_analyzer/log_parser.rb +0 -35
- data/lib/rails_analyzer/record_inserter.rb +0 -39
- data/tasks/test.rake +0 -8
- data/test/log_fragments/fragment_1.log +0 -59
- data/test/log_fragments/fragment_2.log +0 -5
- data/test/log_fragments/fragment_3.log +0 -12
- data/test/log_fragments/fragment_4.log +0 -10
- data/test/log_fragments/fragment_5.log +0 -24
- data/test/log_fragments/merb_1.log +0 -84
- data/test/merb_log_parser_test.rb +0 -39
- data/test/rails_log_parser_test.rb +0 -94
- data/test/record_inserter_test.rb +0 -45
@@ -0,0 +1,180 @@
|
|
1
|
+
module RequestLogAnalyzer::Source
|
2
|
+
# The LogParser class reads log data from a given source and uses a file format definition
|
3
|
+
# to parse all relevent information about requests from the file.
|
4
|
+
#
|
5
|
+
# A FileFormat module should be provided that contains the definitions of the lines that
|
6
|
+
# occur in the log data. The log parser can run in two modes:
|
7
|
+
# - In single line mode, it will emit every detected line as a separate request
|
8
|
+
# - In combined requests mode, it will combine the different lines from the line defintions
|
9
|
+
# into one request, that will then be emitted.
|
10
|
+
#
|
11
|
+
# The combined requests mode gives better information, but can be problematic if the log
|
12
|
+
# file is unordered. This can be the case if data is written to the log file simultaneously
|
13
|
+
# by different mongrel processes. This problem is detected by the parser, but the requests
|
14
|
+
# that are mixed up cannot be parsed. It will emit warnings when this occurs.
|
15
|
+
class LogFile < RequestLogAnalyzer::Source::Base
|
16
|
+
|
17
|
+
attr_reader :source_files
|
18
|
+
|
19
|
+
# Initializes the parser instance.
|
20
|
+
# It will apply the language specific FileFormat module to this instance. It will use the line
|
21
|
+
# definitions in this module to parse any input.
|
22
|
+
def initialize(format, options = {})
|
23
|
+
@line_definitions = {}
|
24
|
+
@options = options
|
25
|
+
@parsed_lines = 0
|
26
|
+
@parsed_requests = 0
|
27
|
+
@skipped_requests = 0
|
28
|
+
@current_io = nil
|
29
|
+
@source_files = options[:source_files]
|
30
|
+
|
31
|
+
# install the file format module (see RequestLogAnalyzer::FileFormat)
|
32
|
+
# and register all the line definitions to the parser
|
33
|
+
self.register_file_format(format)
|
34
|
+
end
|
35
|
+
|
36
|
+
def requests(options = {}, &block)
|
37
|
+
|
38
|
+
case @source_files
|
39
|
+
when IO;
|
40
|
+
puts "Parsing from the standard input. Press CTRL+C to finish."
|
41
|
+
parse_stream(@source_files, options, &block)
|
42
|
+
when String
|
43
|
+
parse_file(@source_files, options, &block)
|
44
|
+
when Array
|
45
|
+
parse_files(@source_files, options, &block)
|
46
|
+
else
|
47
|
+
raise "Unknown source provided"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# Parses a list of consequent files of the same format
|
52
|
+
def parse_files(files, options = {}, &block)
|
53
|
+
files.each { |file| parse_file(file, options, &block) }
|
54
|
+
end
|
55
|
+
|
56
|
+
# Parses a file.
|
57
|
+
# Creates an IO stream for the provided file, and sends it to parse_io for further handling
|
58
|
+
def parse_file(file, options = {}, &block)
|
59
|
+
@progress_handler.call(:started, file) if @progress_handler
|
60
|
+
File.open(file, 'r') { |f| parse_io(f, options, &block) }
|
61
|
+
@progress_handler.call(:finished, file) if @progress_handler
|
62
|
+
end
|
63
|
+
|
64
|
+
def parse_stream(stream, options = {}, &block)
|
65
|
+
parse_io(stream, options, &block)
|
66
|
+
end
|
67
|
+
|
68
|
+
# Finds a log line and then parses the information in the line.
|
69
|
+
# Yields a hash containing the information found.
|
70
|
+
# <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
|
71
|
+
# Yeilds a Hash when it encounters a chunk of information.
|
72
|
+
def parse_io(io, options = {}, &block)
|
73
|
+
|
74
|
+
# parse every line type by default
|
75
|
+
line_types = options[:line_types] || file_format.line_definitions.keys
|
76
|
+
|
77
|
+
# check whether all provided line types are valid
|
78
|
+
unknown = line_types.reject { |line_type| file_format.line_definitions.has_key?(line_type) }
|
79
|
+
raise "Unknown line types: #{unknown.join(', ')}" unless unknown.empty?
|
80
|
+
|
81
|
+
puts "Parsing mode: " + (options[:combined_requests] ? 'combined requests' : 'single lines') if options[:debug]
|
82
|
+
|
83
|
+
@current_io = io
|
84
|
+
@current_io.each_line do |line|
|
85
|
+
|
86
|
+
@progress_handler.call(:progress, @current_io.pos) if @progress_handler && @current_io.kind_of?(File)
|
87
|
+
|
88
|
+
request_data = nil
|
89
|
+
line_types.each do |line_type|
|
90
|
+
line_type_definition = file_format.line_definitions[line_type]
|
91
|
+
break if request_data = line_type_definition.matches(line, @current_io.lineno, self)
|
92
|
+
end
|
93
|
+
|
94
|
+
if request_data
|
95
|
+
@parsed_lines += 1
|
96
|
+
if @options[:combined_requests]
|
97
|
+
update_current_request(request_data, &block)
|
98
|
+
else
|
99
|
+
@parsed_requests +=1
|
100
|
+
yield RequestLogAnalyzer::Request.create(@file_format, request_data)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
|
106
|
+
|
107
|
+
@current_io = nil
|
108
|
+
end
|
109
|
+
|
110
|
+
# Add a block to this method to install a progress handler while parsing
|
111
|
+
def progress=(proc)
|
112
|
+
@progress_handler = proc
|
113
|
+
end
|
114
|
+
|
115
|
+
# Add a block to this method to install a warning handler while parsing
|
116
|
+
def warning=(proc)
|
117
|
+
@warning_handler = proc
|
118
|
+
end
|
119
|
+
|
120
|
+
# This method is called by the parser if it encounteres any problems.
|
121
|
+
# It will call the warning handler. The default controller will pass all warnings to every
|
122
|
+
# aggregator that is registered and running
|
123
|
+
def warn(type, message)
|
124
|
+
@warning_handler.call(type, message, @current_io.lineno) if @warning_handler
|
125
|
+
end
|
126
|
+
|
127
|
+
protected
|
128
|
+
|
129
|
+
# Combines the different lines of a request into a single Request object.
|
130
|
+
# This function is only called in combined requests mode. It will start a new request when
|
131
|
+
# a header line is encountered en will emit the request when a footer line is encountered.
|
132
|
+
#
|
133
|
+
# - Every line that is parsed before a header line is ignored as it cannot be included in
|
134
|
+
# any request. It will emit a :no_current_request warning.
|
135
|
+
# - A header line that is parsed before a request is closed by a footer line, is a sign of
|
136
|
+
# an unprpertly ordered file. All data that is gathered for the request until then is
|
137
|
+
# discarded, the next request is ignored as well and a :unclosed_request warning is
|
138
|
+
# emitted.
|
139
|
+
def update_current_request(request_data, &block)
|
140
|
+
if header_line?(request_data)
|
141
|
+
unless @current_request.nil?
|
142
|
+
if options[:assume_correct_order]
|
143
|
+
@parsed_requests += 1
|
144
|
+
yield @current_request
|
145
|
+
@current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)
|
146
|
+
else
|
147
|
+
@skipped_requests += 1
|
148
|
+
warn(:unclosed_request, "Encountered header line, but previous request was not closed!")
|
149
|
+
@current_request = nil # remove all data that was parsed, skip next request as well.
|
150
|
+
end
|
151
|
+
else
|
152
|
+
@current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)
|
153
|
+
end
|
154
|
+
else
|
155
|
+
unless @current_request.nil?
|
156
|
+
@current_request << request_data
|
157
|
+
if footer_line?(request_data)
|
158
|
+
@parsed_requests += 1
|
159
|
+
yield @current_request
|
160
|
+
@current_request = nil
|
161
|
+
end
|
162
|
+
else
|
163
|
+
@skipped_requests += 1
|
164
|
+
warn(:no_current_request, "Parsebale line found outside of a request!")
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
# Checks whether a given line hash is a header line.
|
170
|
+
def header_line?(hash)
|
171
|
+
file_format.line_definitions[hash[:line_type]].header
|
172
|
+
end
|
173
|
+
|
174
|
+
# Checks whether a given line hash is a footer line.
|
175
|
+
def footer_line?(hash)
|
176
|
+
file_format.line_definitions[hash[:line_type]].footer
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module RequestLogAnalyzer
|
2
|
+
module Tracker
|
3
|
+
|
4
|
+
# Base tracker. All other trackers inherit from this class
|
5
|
+
#
|
6
|
+
# Accepts the following options:
|
7
|
+
# * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
|
8
|
+
# * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
|
9
|
+
# * <tt>:output</tt> Direct output here (defaults to STDOUT)
|
10
|
+
#
|
11
|
+
# For example :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
|
12
|
+
class Base
|
13
|
+
|
14
|
+
attr_reader :options
|
15
|
+
|
16
|
+
def initialize(options ={})
|
17
|
+
@options = options
|
18
|
+
end
|
19
|
+
|
20
|
+
def prepare
|
21
|
+
end
|
22
|
+
|
23
|
+
def update(request)
|
24
|
+
end
|
25
|
+
|
26
|
+
def finalize
|
27
|
+
end
|
28
|
+
|
29
|
+
def should_update?(request)
|
30
|
+
return false if options[:line_type] && !request.has_line_type?(options[:line_type])
|
31
|
+
|
32
|
+
if options[:if].kind_of?(Symbol)
|
33
|
+
return false unless request[options[:if]]
|
34
|
+
elsif options[:if].respond_to?(:call)
|
35
|
+
return false unless options[:if].call(request)
|
36
|
+
end
|
37
|
+
|
38
|
+
if options[:unless].kind_of?(Symbol)
|
39
|
+
return false if request[options[:unless]]
|
40
|
+
elsif options[:unless].respond_to?(:call)
|
41
|
+
return false if options[:unless].call(request)
|
42
|
+
end
|
43
|
+
|
44
|
+
return true
|
45
|
+
end
|
46
|
+
|
47
|
+
def report(output=STDOUT, report_width = 80, color = false)
|
48
|
+
output << self.inspect
|
49
|
+
output << "\n"
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
module RequestLogAnalyzer::Tracker
|
2
|
+
|
3
|
+
# Catagorize requests.
|
4
|
+
# Count and analyze requests for a specific attribute
|
5
|
+
#
|
6
|
+
# Accepts the following options:
|
7
|
+
# * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
|
8
|
+
# * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
|
9
|
+
# * <tt>:title</tt> Title do be displayed above the report.
|
10
|
+
# * <tt>:category</tt> Proc that handles the request categorization.
|
11
|
+
# * <tt>:amount</tt> The amount of lines in the report
|
12
|
+
#
|
13
|
+
# The items in the update request hash are set during the creation of the Duration tracker.
|
14
|
+
#
|
15
|
+
# Example output:
|
16
|
+
# HTTP methods
|
17
|
+
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
18
|
+
# GET ┃ 22248 hits (46.2%) ┃░░░░░░░░░░░░░░░░░
|
19
|
+
# PUT ┃ 13685 hits (28.4%) ┃░░░░░░░░░░░
|
20
|
+
# POST ┃ 11662 hits (24.2%) ┃░░░░░░░░░
|
21
|
+
# DELETE ┃ 512 hits (1.1%) ┃
|
22
|
+
class Category < RequestLogAnalyzer::Tracker::Base
|
23
|
+
|
24
|
+
attr_reader :categories
|
25
|
+
|
26
|
+
def prepare
|
27
|
+
raise "No categorizer set up for category tracker #{self.inspect}" unless options[:category]
|
28
|
+
@categories = {}
|
29
|
+
if options[:all_categories].kind_of?(Enumerable)
|
30
|
+
options[:all_categories].each { |cat| @categories[cat] = 0 }
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def update(request)
|
35
|
+
cat = options[:category].respond_to?(:call) ? options[:category].call(request) : request[options[:category]]
|
36
|
+
if !cat.nil? || options[:nils]
|
37
|
+
@categories[cat] ||= 0
|
38
|
+
@categories[cat] += 1
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def report(output = STDOUT, report_width = 80, color = false)
|
43
|
+
if options[:title]
|
44
|
+
output << "\n#{options[:title]}\n"
|
45
|
+
output << green(('━' * report_width), color) + "\n"
|
46
|
+
end
|
47
|
+
|
48
|
+
if @categories.empty?
|
49
|
+
output << "None found.\n"
|
50
|
+
else
|
51
|
+
sorted_categories = @categories.sort { |a, b| b[1] <=> a[1] }
|
52
|
+
total_hits = sorted_categories.inject(0) { |carry, item| carry + item[1] }
|
53
|
+
sorted_categories = sorted_categories.slice(0...options[:amount]) if options[:amount]
|
54
|
+
|
55
|
+
adjuster = color ? 33 : 24 # justifcation calcultaion is slight different when color codes are inserterted
|
56
|
+
max_cat_length = [sorted_categories.map { |c| c[0].to_s.length }.max, report_width - adjuster].min
|
57
|
+
sorted_categories.each do |(cat, count)|
|
58
|
+
text = "%-#{max_cat_length+1}s┃%7d hits %s" % [cat.to_s[0..max_cat_length], count, (green("(%0.01f%%)", color) % [(count.to_f / total_hits) * 100])]
|
59
|
+
space_left = report_width - (max_cat_length + adjuster + 3)
|
60
|
+
if space_left > 3
|
61
|
+
bar_chars = (space_left * (count.to_f / total_hits)).round
|
62
|
+
output << "%-#{max_cat_length + adjuster}s %s%s" % [text, '┃', '░' * bar_chars] + "\n"
|
63
|
+
else
|
64
|
+
output << text + "\n"
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
module RequestLogAnalyzer::Tracker
|
2
|
+
|
3
|
+
# Analyze the duration of a specific attribute
|
4
|
+
#
|
5
|
+
# Accepts the following options:
|
6
|
+
# * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
|
7
|
+
# * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
|
8
|
+
# * <tt>:title</tt> Title do be displayed above the report
|
9
|
+
# * <tt>:category</tt> Proc that handles request categorization for given fileformat (REQUEST_CATEGORIZER)
|
10
|
+
# * <tt>:duration</tt> The field containing the duration in the request hash.
|
11
|
+
# * <tt>:amount</tt> The amount of lines in the report
|
12
|
+
#
|
13
|
+
# The items in the update request hash are set during the creation of the Duration tracker.
|
14
|
+
#
|
15
|
+
# Example output:
|
16
|
+
# Request duration - top 20 by cumulative time ┃ Hits ┃ Sum. | Avg.
|
17
|
+
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
18
|
+
# EmployeeController#show.html [GET] ┃ 4742 ┃ 4922.56s ┃ 1.04s
|
19
|
+
# EmployeeController#update.html [POST] ┃ 4647 ┃ 2731.23s ┃ 0.59s
|
20
|
+
# EmployeeController#index.html [GET] ┃ 5802 ┃ 1477.32s ┃ 0.25s
|
21
|
+
# .............
|
22
|
+
class Duration < RequestLogAnalyzer::Tracker::Base
|
23
|
+
attr_reader :categories
|
24
|
+
|
25
|
+
def prepare
|
26
|
+
raise "No duration field set up for category tracker #{self.inspect}" unless options[:duration]
|
27
|
+
raise "No categorizer set up for duration tracker #{self.inspect}" unless options[:category]
|
28
|
+
|
29
|
+
@categories = {}
|
30
|
+
end
|
31
|
+
|
32
|
+
def update(request)
|
33
|
+
category = options[:category].respond_to?(:call) ? options[:category].call(request) : request[options[:category]]
|
34
|
+
duration = options[:duration].respond_to?(:call) ? options[:duration].call(request) : request[options[:duration]]
|
35
|
+
|
36
|
+
if !duration.nil? && !category.nil?
|
37
|
+
@categories[category] ||= {:count => 0, :total_duration => 0.0}
|
38
|
+
@categories[category][:count] += 1
|
39
|
+
@categories[category][:total_duration] += duration
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def report_table(output = STDOUT, amount = 10, options = {}, &block)
|
44
|
+
|
45
|
+
top_categories = @categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }.slice(0...amount)
|
46
|
+
max_cat_length = top_categories.map { |a| a[0].length }.max || 0
|
47
|
+
space_left = [options[:report_width] - 33, [max_cat_length + 1, options[:title].length].max].min
|
48
|
+
|
49
|
+
output << "\n"
|
50
|
+
output << "%-#{space_left+1}s┃ Hits ┃ Sum. | Avg." % [options[:title][0...space_left]] + "\n"
|
51
|
+
output << green('━' * options[:report_width], options[:color]) + "\n"
|
52
|
+
|
53
|
+
top_categories.each do |(cat, info)|
|
54
|
+
hits = info[:count]
|
55
|
+
total = "%0.02f" % info[:total_duration]
|
56
|
+
avg = "%0.02f" % (info[:total_duration] / info[:count])
|
57
|
+
output << "%-#{space_left+1}s┃%8d ┃%9ss ┃%9ss" % [cat[0...space_left], hits, total, avg] + "\n"
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def report(output = STDOUT, report_width = 80, color = false)
|
62
|
+
|
63
|
+
options[:title] ||= 'Request duration'
|
64
|
+
options[:report] ||= [:total, :average]
|
65
|
+
options[:top] ||= 20
|
66
|
+
|
67
|
+
options[:report].each do |report|
|
68
|
+
case report
|
69
|
+
when :average
|
70
|
+
report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by average time", :color => color, :report_width => report_width) { |request| request[:total_duration] / request[:count] }
|
71
|
+
when :total
|
72
|
+
report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by cumulative time", :color => color, :report_width => report_width) { |request| request[:total_duration] }
|
73
|
+
when :hits
|
74
|
+
report_table(output, options[:top], :title => "#{options[:title]} - top #{options[:top]} by hits", :color => color, :report_width => report_width) { |request| request[:count] }
|
75
|
+
else
|
76
|
+
output << "Unknown duration report specified\n"
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
module RequestLogAnalyzer::Tracker
|
2
|
+
|
3
|
+
# Determines the average hourly spread of the parsed requests.
|
4
|
+
# This spread is shown in a graph form.
|
5
|
+
#
|
6
|
+
# Accepts the following options:
|
7
|
+
# * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
|
8
|
+
# * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
|
9
|
+
#
|
10
|
+
# Expects the following items in the update request hash
|
11
|
+
# * <tt>:timestamp</tt> in YYYYMMDDHHMMSS format.
|
12
|
+
#
|
13
|
+
# Example output:
|
14
|
+
# Requests graph - average per day per hour
|
15
|
+
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
16
|
+
# 7:00 - 330 hits : ░░░░░░░
|
17
|
+
# 8:00 - 704 hits : ░░░░░░░░░░░░░░░░░
|
18
|
+
# 9:00 - 830 hits : ░░░░░░░░░░░░░░░░░░░░
|
19
|
+
# 10:00 - 822 hits : ░░░░░░░░░░░░░░░░░░░
|
20
|
+
# 11:00 - 823 hits : ░░░░░░░░░░░░░░░░░░░
|
21
|
+
# 12:00 - 729 hits : ░░░░░░░░░░░░░░░░░
|
22
|
+
# 13:00 - 614 hits : ░░░░░░░░░░░░░░
|
23
|
+
# 14:00 - 690 hits : ░░░░░░░░░░░░░░░░
|
24
|
+
# 15:00 - 492 hits : ░░░░░░░░░░░
|
25
|
+
# 16:00 - 355 hits : ░░░░░░░░
|
26
|
+
# 17:00 - 213 hits : ░░░░░
|
27
|
+
# 18:00 - 107 hits : ░░
|
28
|
+
# ................
|
29
|
+
class HourlySpread < RequestLogAnalyzer::Tracker::Base
|
30
|
+
|
31
|
+
attr_reader :first, :last, :request_time_graph
|
32
|
+
|
33
|
+
def prepare
|
34
|
+
options[:field] ||= :timestamp
|
35
|
+
@request_time_graph = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
|
36
|
+
end
|
37
|
+
|
38
|
+
def update(request)
|
39
|
+
request = request.attributes
|
40
|
+
timestamp = request[options[:field]]
|
41
|
+
|
42
|
+
@request_time_graph[timestamp.to_s[8..9].to_i] +=1
|
43
|
+
@first = timestamp if @first.nil? || timestamp < @first
|
44
|
+
@last = timestamp if @last.nil? || timestamp > @last
|
45
|
+
end
|
46
|
+
|
47
|
+
def report(output = STDOUT, report_width = 80, color = false)
|
48
|
+
output << "\n"
|
49
|
+
output << "Requests graph - average per day per hour\n"
|
50
|
+
output << green("━" * report_width, color) + "\n"
|
51
|
+
|
52
|
+
if @request_time_graph == [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
|
53
|
+
output << "None found.\n"
|
54
|
+
return
|
55
|
+
end
|
56
|
+
|
57
|
+
first_date = DateTime.parse(@first.to_s, '%Y%m%d%H%M%S')
|
58
|
+
last_date = DateTime.parse(@last.to_s, '%Y%m%d%H%M%S')
|
59
|
+
days = (@last && @first) ? (last_date - first_date).ceil : 1
|
60
|
+
deviation = @request_time_graph.max / 20
|
61
|
+
deviation = 1 if deviation == 0
|
62
|
+
color_cutoff = 15
|
63
|
+
|
64
|
+
@request_time_graph.each_with_index do |requests, index|
|
65
|
+
display_chars = requests / deviation
|
66
|
+
request_today = requests / days
|
67
|
+
|
68
|
+
if display_chars >= color_cutoff
|
69
|
+
display_chars_string = green(('░' * color_cutoff), color) + red(('░' * (display_chars - color_cutoff)), color)
|
70
|
+
else
|
71
|
+
display_chars_string = green(('░' * display_chars), color)
|
72
|
+
end
|
73
|
+
|
74
|
+
output << "#{index.to_s.rjust(3)}:00 - #{(request_today.to_s + ' hits').ljust(15)} : #{display_chars_string}\n"
|
75
|
+
end
|
76
|
+
output << "\n"
|
77
|
+
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module RequestLogAnalyzer::Tracker
|
2
|
+
|
3
|
+
# Determines the datetime of the first request and the last request
|
4
|
+
# Also determines the amount of days inbetween these.
|
5
|
+
#
|
6
|
+
# Accepts the following options:
|
7
|
+
# * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
|
8
|
+
# * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
|
9
|
+
# * <tt>:field</tt> The timestamp field that is looked at. Defaults to :timestamp.
|
10
|
+
# * <tt>:title</tt> Title do be displayed above the report.
|
11
|
+
#
|
12
|
+
# Expects the following items in the update request hash
|
13
|
+
# * <tt>:timestamp</tt> in YYYYMMDDHHMMSS format.
|
14
|
+
#
|
15
|
+
# Example output:
|
16
|
+
# First request: 2008-07-13 06:25:06
|
17
|
+
# Last request: 2008-07-20 06:18:06
|
18
|
+
# Total time analyzed: 7 days
|
19
|
+
class Timespan < RequestLogAnalyzer::Tracker::Base
|
20
|
+
|
21
|
+
attr_reader :first, :last, :request_time_graph
|
22
|
+
|
23
|
+
def prepare
|
24
|
+
options[:field] ||= :timestamp
|
25
|
+
end
|
26
|
+
|
27
|
+
def update(request)
|
28
|
+
timestamp = request[options[:field]]
|
29
|
+
|
30
|
+
@first = timestamp if @first.nil? || timestamp < @first
|
31
|
+
@last = timestamp if @last.nil? || timestamp > @last
|
32
|
+
end
|
33
|
+
|
34
|
+
def report(output = STDOUT, report_width = 80, color = false)
|
35
|
+
if options[:title]
|
36
|
+
output << "\n#{options[:title]}\n"
|
37
|
+
output << green('━' * options[:title].length, color) + "\n"
|
38
|
+
end
|
39
|
+
|
40
|
+
first_date = DateTime.parse(@first.to_s, '%Y%m%d%H%M%S') rescue nil
|
41
|
+
last_date = DateTime.parse(@last.to_s, '%Y%m%d%H%M%S') rescue nil
|
42
|
+
|
43
|
+
if @last && @first
|
44
|
+
days = (@last && @first) ? (last_date - first_date).ceil : 1
|
45
|
+
|
46
|
+
output << "First request: #{first_date.strftime('%Y-%m-%d %H:%M:%I')}\n"
|
47
|
+
output << "Last request: #{last_date.strftime('%Y-%m-%d %H:%M:%I')}\n"
|
48
|
+
output << "Total time analyzed: #{days} days\n"
|
49
|
+
end
|
50
|
+
output << "\n"
|
51
|
+
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
|
3
|
+
describe RequestLogAnalyzer::FileFormat, :format_definition do
|
4
|
+
|
5
|
+
before(:each) do
|
6
|
+
@first_file_format = Class.new(RequestLogAnalyzer::FileFormat)
|
7
|
+
@second_file_format = Class.new(RequestLogAnalyzer::FileFormat)
|
8
|
+
end
|
9
|
+
|
10
|
+
it "should specify lines with a hash" do
|
11
|
+
|
12
|
+
@first_file_format.new.should have(0).line_definitions
|
13
|
+
|
14
|
+
@first_file_format.format_definition do |line|
|
15
|
+
line.hash_test :regexp => /test/, :captures => []
|
16
|
+
end
|
17
|
+
|
18
|
+
@format_instance = @first_file_format.new
|
19
|
+
@format_instance.should have(1).line_definitions
|
20
|
+
@format_instance.line_definitions[:hash_test].should_not be_nil
|
21
|
+
end
|
22
|
+
|
23
|
+
it "should specift lines directly" do
|
24
|
+
@first_file_format.new.should have(0).line_definitions
|
25
|
+
|
26
|
+
@first_file_format.format_definition.direct_test do |line|
|
27
|
+
line.regexp = /test/
|
28
|
+
end
|
29
|
+
|
30
|
+
@first_file_format.new.line_definitions[:direct_test].should_not be_nil
|
31
|
+
end
|
32
|
+
|
33
|
+
it "specify lines with a block" do
|
34
|
+
|
35
|
+
@first_file_format.new.should have(0).line_definitions
|
36
|
+
|
37
|
+
@first_file_format.format_definition do |format|
|
38
|
+
format.block_test do |line|
|
39
|
+
line.regexp = /test/
|
40
|
+
line.captures = []
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
@first_file_format.new.line_definitions[:block_test].should_not be_nil
|
45
|
+
end
|
46
|
+
|
47
|
+
it "should define lines only for itself" do
|
48
|
+
|
49
|
+
@first_file_format.format_definition do |line|
|
50
|
+
line.first_test :regexp => /test/, :captures => []
|
51
|
+
end
|
52
|
+
|
53
|
+
@second_file_format.format_definition do |line|
|
54
|
+
line.second_test :regexp => /test/, :captures => []
|
55
|
+
end
|
56
|
+
|
57
|
+
@first_file_format.new.should have(1).line_definitions
|
58
|
+
@first_file_format.new.line_definitions[:first_test].should_not be_nil
|
59
|
+
@second_file_format.new.should have(1).line_definitions
|
60
|
+
@second_file_format.new.line_definitions[:second_test].should_not be_nil
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
describe RequestLogAnalyzer::FileFormat, :load do
|
65
|
+
|
66
|
+
include RequestLogAnalyzerSpecHelper
|
67
|
+
|
68
|
+
it "should load a predefined file format from the /file_format dir" do
|
69
|
+
@file_format = RequestLogAnalyzer::FileFormat.load(:rails)
|
70
|
+
@file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Rails)
|
71
|
+
end
|
72
|
+
|
73
|
+
it "should load a provided format file" do
|
74
|
+
@file_format = RequestLogAnalyzer::FileFormat.load(format_file(:spec_format))
|
75
|
+
@file_format.should be_kind_of(SpecFormat)
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
class SpecFormat < RequestLogAnalyzer::FileFormat
|
2
|
+
|
3
|
+
format_definition.first do |line|
|
4
|
+
line.header = true
|
5
|
+
line.teaser = /processing /
|
6
|
+
line.regexp = /processing request (\d+)/
|
7
|
+
line.captures = [{ :name => :request_no, :type => :integer, :anonymize => :slightly }]
|
8
|
+
end
|
9
|
+
|
10
|
+
format_definition.test do |line|
|
11
|
+
line.teaser = /testing /
|
12
|
+
line.regexp = /testing is (\w+)/
|
13
|
+
line.captures = [{ :name => :test_capture, :type => :string, :anonymize => true}]
|
14
|
+
end
|
15
|
+
|
16
|
+
format_definition.last do |line|
|
17
|
+
line.footer = true
|
18
|
+
line.teaser = /finishing /
|
19
|
+
line.regexp = /finishing request (\d+)/
|
20
|
+
line.captures = [{ :name => :request_no, :type => :integer}]
|
21
|
+
end
|
22
|
+
|
23
|
+
report do |analyze|
|
24
|
+
analyze.category :test_capture, :title => 'What is testing exactly?'
|
25
|
+
end
|
26
|
+
end
|