request-log-analyzer 1.13.1 → 1.13.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/bin/console +17 -0
- data/lib/cli/command_line_arguments.rb +29 -36
- data/lib/cli/database_console.rb +1 -3
- data/lib/cli/database_console_init.rb +11 -11
- data/lib/cli/progressbar.rb +30 -32
- data/lib/cli/tools.rb +20 -23
- data/lib/request_log_analyzer.rb +8 -8
- data/lib/request_log_analyzer/aggregator.rb +4 -7
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +10 -13
- data/lib/request_log_analyzer/aggregator/echo.rb +5 -7
- data/lib/request_log_analyzer/aggregator/summarizer.rb +15 -18
- data/lib/request_log_analyzer/class_level_inheritable_attributes.rb +23 -0
- data/lib/request_log_analyzer/controller.rb +36 -42
- data/lib/request_log_analyzer/database.rb +4 -6
- data/lib/request_log_analyzer/database/base.rb +39 -41
- data/lib/request_log_analyzer/database/connection.rb +8 -10
- data/lib/request_log_analyzer/database/request.rb +1 -3
- data/lib/request_log_analyzer/database/source.rb +0 -2
- data/lib/request_log_analyzer/database/warning.rb +4 -6
- data/lib/request_log_analyzer/file_format.rb +46 -49
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +15 -19
- data/lib/request_log_analyzer/file_format/apache.rb +42 -45
- data/lib/request_log_analyzer/file_format/delayed_job.rb +13 -15
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job3.rb +5 -8
- data/lib/request_log_analyzer/file_format/delayed_job4.rb +5 -8
- data/lib/request_log_analyzer/file_format/haproxy.rb +44 -48
- data/lib/request_log_analyzer/file_format/merb.rb +13 -17
- data/lib/request_log_analyzer/file_format/mysql.rb +21 -25
- data/lib/request_log_analyzer/file_format/nginx.rb +0 -2
- data/lib/request_log_analyzer/file_format/oink.rb +30 -31
- data/lib/request_log_analyzer/file_format/postgresql.rb +11 -15
- data/lib/request_log_analyzer/file_format/rack.rb +0 -2
- data/lib/request_log_analyzer/file_format/rails.rb +100 -104
- data/lib/request_log_analyzer/file_format/rails3.rb +19 -23
- data/lib/request_log_analyzer/file_format/rails_development.rb +0 -1
- data/lib/request_log_analyzer/file_format/w3c.rb +16 -18
- data/lib/request_log_analyzer/filter.rb +0 -2
- data/lib/request_log_analyzer/filter/anonymize.rb +4 -7
- data/lib/request_log_analyzer/filter/field.rb +3 -6
- data/lib/request_log_analyzer/filter/timespan.rb +2 -6
- data/lib/request_log_analyzer/line_definition.rb +16 -19
- data/lib/request_log_analyzer/log_processor.rb +10 -14
- data/lib/request_log_analyzer/mailer.rb +9 -12
- data/lib/request_log_analyzer/output.rb +12 -14
- data/lib/request_log_analyzer/output/fixed_width.rb +21 -28
- data/lib/request_log_analyzer/output/html.rb +11 -14
- data/lib/request_log_analyzer/request.rb +53 -33
- data/lib/request_log_analyzer/source.rb +2 -5
- data/lib/request_log_analyzer/source/log_parser.rb +9 -16
- data/lib/request_log_analyzer/tracker.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +4 -6
- data/lib/request_log_analyzer/tracker/frequency.rb +9 -11
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -11
- data/lib/request_log_analyzer/tracker/numeric_value.rb +40 -44
- data/lib/request_log_analyzer/tracker/timespan.rb +5 -8
- data/lib/request_log_analyzer/tracker/traffic.rb +8 -10
- data/lib/request_log_analyzer/version.rb +1 -1
- data/request-log-analyzer.gemspec +6 -6
- data/spec/integration/command_line_usage_spec.rb +33 -33
- data/spec/integration/mailer_spec.rb +181 -185
- data/spec/integration/munin_plugins_rails_spec.rb +20 -20
- data/spec/integration/scout_spec.rb +40 -41
- data/spec/lib/helpers.rb +8 -9
- data/spec/lib/macros.rb +2 -4
- data/spec/lib/matchers.rb +20 -25
- data/spec/lib/mocks.rb +10 -11
- data/spec/lib/testing_format.rb +8 -10
- data/spec/spec_helper.rb +5 -1
- data/spec/unit/aggregator/database_inserter_spec.rb +23 -23
- data/spec/unit/aggregator/summarizer_spec.rb +7 -7
- data/spec/unit/controller/controller_spec.rb +14 -14
- data/spec/unit/controller/log_processor_spec.rb +3 -3
- data/spec/unit/database/base_class_spec.rb +36 -37
- data/spec/unit/database/connection_spec.rb +10 -10
- data/spec/unit/database/database_spec.rb +11 -11
- data/spec/unit/file_format/amazon_s3_format_spec.rb +66 -62
- data/spec/unit/file_format/apache_format_spec.rb +57 -52
- data/spec/unit/file_format/common_regular_expressions_spec.rb +18 -21
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job2_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job3_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job4_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job_format_spec.rb +12 -12
- data/spec/unit/file_format/file_format_api_spec.rb +19 -19
- data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
- data/spec/unit/file_format/haproxy_format_spec.rb +53 -49
- data/spec/unit/file_format/inheritance_spec.rb +13 -0
- data/spec/unit/file_format/line_definition_spec.rb +35 -33
- data/spec/unit/file_format/merb_format_spec.rb +13 -11
- data/spec/unit/file_format/mysql_format_spec.rb +24 -24
- data/spec/unit/file_format/oink_format_spec.rb +29 -29
- data/spec/unit/file_format/postgresql_format_spec.rb +9 -9
- data/spec/unit/file_format/rack_format_spec.rb +36 -31
- data/spec/unit/file_format/rails3_format_spec.rb +46 -46
- data/spec/unit/file_format/rails_format_spec.rb +52 -53
- data/spec/unit/file_format/w3c_format_spec.rb +27 -24
- data/spec/unit/filter/anonymize_filter_spec.rb +7 -7
- data/spec/unit/filter/field_filter_spec.rb +26 -26
- data/spec/unit/filter/filter_spec.rb +4 -4
- data/spec/unit/filter/timespan_filter_spec.rb +22 -22
- data/spec/unit/mailer_spec.rb +21 -21
- data/spec/unit/request_spec.rb +29 -29
- data/spec/unit/source/log_parser_spec.rb +5 -5
- data/spec/unit/tracker/duration_tracker_spec.rb +23 -23
- data/spec/unit/tracker/frequency_tracker_spec.rb +29 -30
- data/spec/unit/tracker/hourly_spread_spec.rb +35 -35
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +71 -72
- data/spec/unit/tracker/timespan_tracker_spec.rb +31 -31
- data/spec/unit/tracker/tracker_api_spec.rb +43 -44
- data/spec/unit/tracker/traffic_tracker_spec.rb +7 -7
- metadata +38 -35
@@ -1,22 +1,19 @@
|
|
1
1
|
# coding: utf-8
|
2
2
|
module RequestLogAnalyzer::Output
|
3
|
-
|
4
3
|
# Fixed Width output class.
|
5
4
|
# Outputs a fixed width ASCII or UF8 report.
|
6
5
|
class FixedWidth < Base
|
7
|
-
|
8
6
|
# Mixin module. Will disable any colorizing.
|
9
7
|
module Monochrome
|
10
|
-
def colorize(text, *
|
8
|
+
def colorize(text, *_options)
|
11
9
|
text
|
12
10
|
end
|
13
11
|
end
|
14
12
|
|
15
13
|
# Colorize module
|
16
14
|
module Color
|
17
|
-
|
18
|
-
|
19
|
-
COLORS = { :black => 0, :blue => 4, :green => 2, :cyan => 6, :red => 1, :purple => 5, :brown => 3, :white => 7 }
|
15
|
+
STYLES = { normal: 0, bold: 1, underscore: 4, blink: 5, inverse: 7, concealed: 8 }
|
16
|
+
COLORS = { black: 0, blue: 4, green: 2, cyan: 6, red: 1, purple: 5, brown: 3, white: 7 }
|
20
17
|
|
21
18
|
# Colorize text
|
22
19
|
# <tt>text</tt> The text to colorize
|
@@ -28,36 +25,34 @@ module RequestLogAnalyzer::Output
|
|
28
25
|
#
|
29
26
|
# Returns ASCII colored string
|
30
27
|
def colorize(text, *options)
|
31
|
-
|
32
28
|
font_style = ''
|
33
29
|
foreground_color = '0'
|
34
30
|
background_color = ''
|
35
31
|
|
36
32
|
options.each do |option|
|
37
|
-
if option.
|
33
|
+
if option.is_a?(Symbol)
|
38
34
|
foreground_color = "3#{COLORS[option]}" if COLORS.include?(option)
|
39
35
|
font_style = "#{STYLES[option]};" if STYLES.include?(option)
|
40
|
-
elsif option.
|
36
|
+
elsif option.is_a?(Hash)
|
41
37
|
option.each do |key, value|
|
42
38
|
case key
|
43
|
-
when :color
|
44
|
-
when :background
|
45
|
-
when :on
|
46
|
-
when :style
|
39
|
+
when :color then foreground_color = "3#{COLORS[value]}" if COLORS.include?(value)
|
40
|
+
when :background then background_color = "4#{COLORS[value]};" if COLORS.include?(value)
|
41
|
+
when :on then background_color = "4#{COLORS[value]};" if COLORS.include?(value)
|
42
|
+
when :style then font_style = "#{STYLES[value]};" if STYLES.include?(value)
|
47
43
|
end
|
48
44
|
end
|
49
45
|
end
|
50
46
|
end
|
51
|
-
|
47
|
+
"\e[#{background_color}#{font_style}#{foreground_color}m#{text}\e[0m"
|
52
48
|
end
|
53
|
-
|
54
49
|
end
|
55
50
|
|
56
51
|
attr_reader :characters
|
57
52
|
|
58
53
|
CHARACTERS = {
|
59
|
-
:
|
60
|
-
:
|
54
|
+
ascii: { horizontal_line: '-', vertical_line: '|', block: '=' },
|
55
|
+
utf: { horizontal_line: '━', vertical_line: '┃', block: '░' }
|
61
56
|
}
|
62
57
|
|
63
58
|
# Initialize a report
|
@@ -82,7 +77,7 @@ module RequestLogAnalyzer::Output
|
|
82
77
|
@io << str
|
83
78
|
end
|
84
79
|
|
85
|
-
|
80
|
+
alias_method :<<, :print
|
86
81
|
|
87
82
|
# Write a string to the output object with a newline at the end.
|
88
83
|
# <tt>str</tt> The string to write.
|
@@ -116,8 +111,8 @@ module RequestLogAnalyzer::Output
|
|
116
111
|
|
117
112
|
# Generate a header for a report
|
118
113
|
def header
|
119
|
-
if io.
|
120
|
-
puts colorize(
|
114
|
+
if io.is_a?(File)
|
115
|
+
puts colorize('Request-log-analyzer summary report', :white, :bold)
|
121
116
|
line(:green)
|
122
117
|
puts "Version #{RequestLogAnalyzer::VERSION} - written by Willem van Bergen and Bart ten Brinke"
|
123
118
|
puts "Website: #{link('http://github.com/wvanbergen/request-log-analyzer')}"
|
@@ -127,7 +122,7 @@ module RequestLogAnalyzer::Output
|
|
127
122
|
# Generate a footer for a report
|
128
123
|
def footer
|
129
124
|
puts
|
130
|
-
puts
|
125
|
+
puts 'Need an expert to analyze your application?'
|
131
126
|
puts "Mail to #{link('contact@railsdoctors.com')} or visit us at #{link('http://railsdoctors.com')}."
|
132
127
|
line(:green)
|
133
128
|
puts "Thanks for using #{colorize('request-log-analyzer', :white, :bold)}!"
|
@@ -136,13 +131,12 @@ module RequestLogAnalyzer::Output
|
|
136
131
|
# Generate a report table and push it into the output object.
|
137
132
|
# <tt>*colums<tt> Columns hash
|
138
133
|
# <tt>&block</tt>: A block yeilding the rows.
|
139
|
-
def table(*columns, &
|
140
|
-
|
141
|
-
rows = Array.new
|
134
|
+
def table(*columns, &_block)
|
135
|
+
rows = []
|
142
136
|
yield(rows)
|
143
137
|
|
144
138
|
# determine maximum cell widths
|
145
|
-
max_cell_widths = rows.
|
139
|
+
max_cell_widths = rows.reduce(Array.new(columns.length, 0)) do |result, row|
|
146
140
|
lengths = row.map { |column| column.to_s.length }
|
147
141
|
result.each_with_index { |length, index| result[index] = ([length, lengths[index]].max rescue length) }
|
148
142
|
end
|
@@ -164,7 +158,7 @@ module RequestLogAnalyzer::Output
|
|
164
158
|
end
|
165
159
|
|
166
160
|
if column_widths.include?(nil)
|
167
|
-
width_left = options[:width] - ((columns.length - 1) * (style[:cell_separator] ? 3 : 1)) - column_widths.compact.
|
161
|
+
width_left = options[:width] - ((columns.length - 1) * (style[:cell_separator] ? 3 : 1)) - column_widths.compact.reduce(0) { |sum, col| sum + col }
|
168
162
|
column_widths[column_widths.index(nil)] = width_left
|
169
163
|
end
|
170
164
|
|
@@ -214,6 +208,5 @@ module RequestLogAnalyzer::Output
|
|
214
208
|
puts row_values.join(style[:cell_separator] ? " #{characters[:vertical_line]} " : ' ')
|
215
209
|
end
|
216
210
|
end
|
217
|
-
|
218
211
|
end
|
219
|
-
end
|
212
|
+
end
|
@@ -1,8 +1,6 @@
|
|
1
1
|
module RequestLogAnalyzer::Output
|
2
|
-
|
3
2
|
# HTML Output class. Generated a HTML-formatted report, including CSS.
|
4
3
|
class HTML < Base
|
5
|
-
|
6
4
|
# def initialize(io, options = {})
|
7
5
|
# super(io, options)
|
8
6
|
# end
|
@@ -20,7 +18,7 @@ module RequestLogAnalyzer::Output
|
|
20
18
|
@io << str
|
21
19
|
end
|
22
20
|
|
23
|
-
|
21
|
+
alias_method :<<, :print
|
24
22
|
|
25
23
|
# Put a string with newline
|
26
24
|
def puts(str = '')
|
@@ -34,7 +32,7 @@ module RequestLogAnalyzer::Output
|
|
34
32
|
|
35
33
|
# Render a single line
|
36
34
|
# <tt>*font</tt> The font.
|
37
|
-
def line(*
|
35
|
+
def line(*_font)
|
38
36
|
@io.puts(tag(:hr))
|
39
37
|
end
|
40
38
|
|
@@ -43,17 +41,17 @@ module RequestLogAnalyzer::Output
|
|
43
41
|
# <tt>url</tt> The url to link to.
|
44
42
|
def link(text, url = nil)
|
45
43
|
url = text if url.nil?
|
46
|
-
tag(:a, text, :
|
44
|
+
tag(:a, text, href: url)
|
47
45
|
end
|
48
46
|
|
49
47
|
# Generate a report table in HTML and push it into the output object.
|
50
48
|
# <tt>*colums<tt> Columns hash
|
51
49
|
# <tt>&block</tt>: A block yeilding the rows.
|
52
|
-
def table(*columns, &
|
53
|
-
rows =
|
50
|
+
def table(*columns, &_block)
|
51
|
+
rows = []
|
54
52
|
yield(rows)
|
55
53
|
|
56
|
-
@io << tag(:table,
|
54
|
+
@io << tag(:table, class: 'rla-report-table', cellspacing: 0) do |content|
|
57
55
|
if table_has_header?(columns)
|
58
56
|
content << tag(:tr) do
|
59
57
|
columns.map { |col| tag(:th, col[:title]) }.join("\n")
|
@@ -65,21 +63,20 @@ module RequestLogAnalyzer::Output
|
|
65
63
|
odd = !odd
|
66
64
|
content << tag(:tr) do
|
67
65
|
if odd
|
68
|
-
row.map { |cell| tag(:td, cell, :
|
66
|
+
row.map { |cell| tag(:td, cell, class: 'alt') }.join("\n")
|
69
67
|
else
|
70
68
|
row.map { |cell| tag(:td, cell) }.join("\n")
|
71
69
|
end
|
72
70
|
end
|
73
71
|
end
|
74
72
|
end
|
75
|
-
|
76
73
|
end
|
77
74
|
|
78
75
|
# Genrate HTML header and associated stylesheet
|
79
76
|
def header
|
80
77
|
@io.content_type = content_type if @io.respond_to?(:content_type)
|
81
78
|
|
82
|
-
@io <<
|
79
|
+
@io << '<html>'
|
83
80
|
@io << tag(:head) do |headers|
|
84
81
|
headers << tag(:title, 'Request-log-analyzer report')
|
85
82
|
headers << tag(:style, '
|
@@ -111,7 +108,7 @@ module RequestLogAnalyzer::Output
|
|
111
108
|
|
112
109
|
caption {
|
113
110
|
padding: 0 0 5px 0;
|
114
|
-
width: 700px;
|
111
|
+
width: 700px;
|
115
112
|
font: italic 11px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
116
113
|
text-align: right;
|
117
114
|
}
|
@@ -142,7 +139,7 @@ module RequestLogAnalyzer::Output
|
|
142
139
|
background: #F5FAFA;
|
143
140
|
color: #797268;
|
144
141
|
}
|
145
|
-
', :
|
142
|
+
', type: 'text/css')
|
146
143
|
end
|
147
144
|
@io << '<body>'
|
148
145
|
@io << tag(:h1, 'Request-log-analyzer summary report')
|
@@ -176,7 +173,7 @@ module RequestLogAnalyzer::Output
|
|
176
173
|
"<#{tag}#{attributes} />"
|
177
174
|
else
|
178
175
|
if content.class == Float
|
179
|
-
"<#{tag}#{attributes}><div class='color_bar' style=\"width:#{(content*200).floor}px;\"/></#{tag}>"
|
176
|
+
"<#{tag}#{attributes}><div class='color_bar' style=\"width:#{(content * 200).floor}px;\"/></#{tag}>"
|
180
177
|
else
|
181
178
|
"<#{tag}#{attributes}>#{content}</#{tag}>"
|
182
179
|
end
|
@@ -1,5 +1,4 @@
|
|
1
1
|
module RequestLogAnalyzer
|
2
|
-
|
3
2
|
# The Request class represents a parsed request from the log file.
|
4
3
|
# Instances are created by the LogParser and are passed to the different aggregators, so they
|
5
4
|
# can do their aggregating work.
|
@@ -8,9 +7,7 @@ module RequestLogAnalyzer
|
|
8
7
|
# Request#first(field_name) returns the first (only) value corresponding to the given field
|
9
8
|
# Request#every(field_name) returns all values corresponding to the given field name as array.
|
10
9
|
class Request
|
11
|
-
|
12
10
|
module Converters
|
13
|
-
|
14
11
|
# Default converter function, which converts the parsed strings to a native Ruby type
|
15
12
|
# using the type indication in the line definition. It will use a custom connverter
|
16
13
|
# method if one is available.
|
@@ -20,24 +17,47 @@ module RequestLogAnalyzer
|
|
20
17
|
send(custom_converter_method, value, capture_definition)
|
21
18
|
end
|
22
19
|
|
23
|
-
def convert_string(value,
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
def
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
20
|
+
def convert_string(value, _capture_definition)
|
21
|
+
value
|
22
|
+
end
|
23
|
+
|
24
|
+
def convert_float(value, _capture_definition)
|
25
|
+
value.to_f
|
26
|
+
end
|
27
|
+
|
28
|
+
def convert_decimal(value, _capture_definition)
|
29
|
+
value.to_f
|
30
|
+
end
|
31
|
+
|
32
|
+
def convert_int(value, _capture_definition)
|
33
|
+
value.to_i
|
34
|
+
end
|
35
|
+
|
36
|
+
def convert_integer(value, _capture_definition)
|
37
|
+
value.to_i
|
38
|
+
end
|
39
|
+
|
40
|
+
def convert_sym(value, _capture_definition)
|
41
|
+
value.to_sym
|
42
|
+
end
|
43
|
+
|
44
|
+
def convert_symbol(value, _capture_definition)
|
45
|
+
value.to_sym
|
46
|
+
end
|
47
|
+
|
48
|
+
def convert_nillable_string(value, _definition)
|
49
|
+
value == '-' ? nil : value
|
50
|
+
end
|
51
|
+
|
32
52
|
# This function can be overridden to rewrite the path for better categorization in the
|
33
53
|
# reports.
|
34
|
-
def convert_path(value,
|
54
|
+
def convert_path(value, _definition)
|
35
55
|
value
|
36
56
|
end
|
37
57
|
|
38
58
|
# Converts :eval field, which should evaluate to a hash.
|
39
|
-
def convert_eval(value,
|
40
|
-
eval(sanitize_parameters(value)).
|
59
|
+
def convert_eval(value, _capture_definition)
|
60
|
+
eval(sanitize_parameters(value)).reduce({}) { |h, (k, v)| h[k.to_sym] = v; h }
|
41
61
|
rescue
|
42
62
|
nil
|
43
63
|
end
|
@@ -51,7 +71,7 @@ module RequestLogAnalyzer
|
|
51
71
|
# Slow default method to parse timestamps.
|
52
72
|
# Reimplement this function in a file format specific Request class
|
53
73
|
# to improve the timestamp parsing speed.
|
54
|
-
def convert_timestamp(value,
|
74
|
+
def convert_timestamp(value, _capture_definition)
|
55
75
|
DateTime.parse(value).strftime('%Y%m%d%H%M%S').to_i
|
56
76
|
end
|
57
77
|
|
@@ -59,13 +79,13 @@ module RequestLogAnalyzer
|
|
59
79
|
def convert_traffic(value, capture_definition)
|
60
80
|
case capture_definition[:unit]
|
61
81
|
when nil, :b, :B, :byte then value.to_i
|
62
|
-
when :GB, :G, :gigabyte then (value.to_f *
|
63
|
-
when :GiB, :gibibyte then (value.to_f * (2
|
64
|
-
when :MB, :M, :megabyte then (value.to_f *
|
65
|
-
when :MiB, :mebibyte then (value.to_f * (2
|
82
|
+
when :GB, :G, :gigabyte then (value.to_f * 1_000_000_000).round
|
83
|
+
when :GiB, :gibibyte then (value.to_f * (2**30)).round
|
84
|
+
when :MB, :M, :megabyte then (value.to_f * 1_000_000).round
|
85
|
+
when :MiB, :mebibyte then (value.to_f * (2**20)).round
|
66
86
|
when :KB, :K, :kilobyte, :kB then (value.to_f * 1000).round
|
67
|
-
when :KiB, :kibibyte then (value.to_f * (2
|
68
|
-
else
|
87
|
+
when :KiB, :kibibyte then (value.to_f * (2**10)).round
|
88
|
+
else fail 'Unknown traffic unit'
|
69
89
|
end
|
70
90
|
end
|
71
91
|
|
@@ -73,14 +93,14 @@ module RequestLogAnalyzer
|
|
73
93
|
def convert_duration(value, capture_definition)
|
74
94
|
case capture_definition[:unit]
|
75
95
|
when nil, :sec, :s then value.to_f
|
76
|
-
when :microsec, :musec then value.to_f /
|
96
|
+
when :microsec, :musec then value.to_f / 1_000_000.0
|
77
97
|
when :msec, :millisec then value.to_f / 1000.0
|
78
|
-
else
|
98
|
+
else fail 'Unknown duration unit'
|
79
99
|
end
|
80
100
|
end
|
81
|
-
|
101
|
+
|
82
102
|
# Convert an epoch to an integer
|
83
|
-
def convert_epoch(value,
|
103
|
+
def convert_epoch(value, _capture_definition)
|
84
104
|
Time.at(value.to_i).strftime('%Y%m%d%H%M%S').to_i
|
85
105
|
end
|
86
106
|
end
|
@@ -101,16 +121,16 @@ module RequestLogAnalyzer
|
|
101
121
|
# Creates a new request that was parsed from the log with the given FileFormat. The hashes
|
102
122
|
# that are passed to this function are added as lines to this request.
|
103
123
|
def self.create(file_format, *hashes)
|
104
|
-
request =
|
124
|
+
request = new(file_format)
|
105
125
|
hashes.flatten.each { |hash| request << hash }
|
106
|
-
|
126
|
+
request
|
107
127
|
end
|
108
128
|
|
109
129
|
# Adds another line to the request when it is parsed in the LogParser.
|
110
130
|
#
|
111
131
|
# The line should be provided as a hash with the attributes line_definition, :captures,
|
112
132
|
# :lineno and :source set. This function is called from LogParser.
|
113
|
-
def add_parsed_line
|
133
|
+
def add_parsed_line(parsed_line)
|
114
134
|
value_hash = parsed_line[:line_definition].convert_captured_values(parsed_line[:captures], self)
|
115
135
|
value_hash[:line_type] = parsed_line[:line_definition].name
|
116
136
|
value_hash[:lineno] = parsed_line[:lineno]
|
@@ -148,10 +168,10 @@ module RequestLogAnalyzer
|
|
148
168
|
# Checks whether the given line type was parsed from the log file for this request
|
149
169
|
def has_line_type?(line_type)
|
150
170
|
return true if @lines.length == 1 && @lines[0][:line_type] == line_type.to_sym
|
151
|
-
@lines.
|
171
|
+
@lines.find { |l| l[:line_type] == line_type.to_sym }
|
152
172
|
end
|
153
173
|
|
154
|
-
|
174
|
+
alias_method :=~, :has_line_type?
|
155
175
|
|
156
176
|
# Returns the value that was captured for the "field" of this request.
|
157
177
|
# This function will return the first value that was captured if the field
|
@@ -160,11 +180,11 @@ module RequestLogAnalyzer
|
|
160
180
|
@attributes[field]
|
161
181
|
end
|
162
182
|
|
163
|
-
|
183
|
+
alias_method :[], :first
|
164
184
|
|
165
185
|
# Returns an array of all the "field" values that were captured for this request
|
166
186
|
def every(field)
|
167
|
-
@lines.
|
187
|
+
@lines.reduce([]) { |result, fields| result << fields[field] if fields.key?(field); result }
|
168
188
|
end
|
169
189
|
|
170
190
|
# Returns true if this request does not yet contain any parsed lines. This should only occur
|
@@ -7,13 +7,11 @@
|
|
7
7
|
# - The base class for all sources is RequestLogAnalyzer::Source::Base. All source classes should inherit from this class.
|
8
8
|
# - Currently, RequestLogAnalyzer::Source::LogParser is the only implemented source.
|
9
9
|
module RequestLogAnalyzer::Source
|
10
|
-
|
11
10
|
# The base Source class. All other sources should inherit from this class.
|
12
11
|
#
|
13
12
|
# A source implememtation should at least implement the each_request method, which should yield
|
14
13
|
# RequestLogAnalyzer::Request instances that will be fed through the pipleine.
|
15
14
|
class Base
|
16
|
-
|
17
15
|
# A hash of options
|
18
16
|
attr_reader :options
|
19
17
|
|
@@ -52,15 +50,14 @@ module RequestLogAnalyzer::Source
|
|
52
50
|
# This function is called to actually produce the requests that will be send into the pipeline.
|
53
51
|
# The implementation should yield instances of RequestLogAnalyzer::Request.
|
54
52
|
# <tt>options</tt>:: A Hash of options that can be used in the implementation.
|
55
|
-
def each_request(
|
56
|
-
|
53
|
+
def each_request(_options = {}, &_block) # :yields: request
|
54
|
+
true
|
57
55
|
end
|
58
56
|
|
59
57
|
# This function is called after RequestLogAnalyzer::Source::Base#each_request finished. Any code to
|
60
58
|
# wrap up, free resources, etc. can be put in this method.
|
61
59
|
def finalize
|
62
60
|
end
|
63
|
-
|
64
61
|
end
|
65
62
|
end
|
66
63
|
|
@@ -1,5 +1,4 @@
|
|
1
1
|
module RequestLogAnalyzer::Source
|
2
|
-
|
3
2
|
# The LogParser class reads log data from a given source and uses a file format definition
|
4
3
|
# to parse all relevent information about requests from the file. A FileFormat module should
|
5
4
|
# be provided that contains the definitions of the lines that occur in the log data.
|
@@ -10,7 +9,6 @@ module RequestLogAnalyzer::Source
|
|
10
9
|
# parser. It will emit warnings when this occurs. LogParser supports multiple parse strategies
|
11
10
|
# that deal differently with this problem.
|
12
11
|
class LogParser < Base
|
13
|
-
|
14
12
|
include Enumerable
|
15
13
|
|
16
14
|
# The maximum number of bytes to read from a line.
|
@@ -51,7 +49,7 @@ module RequestLogAnalyzer::Source
|
|
51
49
|
|
52
50
|
@options[:parse_strategy] ||= DEFAULT_PARSE_STRATEGY
|
53
51
|
unless PARSE_STRATEGIES.include?(@options[:parse_strategy])
|
54
|
-
|
52
|
+
fail "Unknown parse strategy: #{@options[@parse_strategy]}"
|
55
53
|
end
|
56
54
|
end
|
57
55
|
|
@@ -68,11 +66,10 @@ module RequestLogAnalyzer::Source
|
|
68
66
|
# that will be yielded. The actual parsing occurs in the parse_io method.
|
69
67
|
# <tt>options</tt>:: A Hash of options that will be pased to parse_io.
|
70
68
|
def each_request(options = {}, &block) # :yields: :request, request
|
71
|
-
|
72
69
|
case @source_files
|
73
70
|
when IO
|
74
71
|
if @source_files == $stdin
|
75
|
-
puts
|
72
|
+
puts 'Parsing from the standard input. Press CTRL+C to finish.' # FIXME: not here
|
76
73
|
end
|
77
74
|
parse_stream(@source_files, options, &block)
|
78
75
|
when String
|
@@ -80,7 +77,7 @@ module RequestLogAnalyzer::Source
|
|
80
77
|
when Array
|
81
78
|
parse_files(@source_files, options, &block)
|
82
79
|
else
|
83
|
-
|
80
|
+
fail 'Unknown source provided'
|
84
81
|
end
|
85
82
|
end
|
86
83
|
|
@@ -98,13 +95,13 @@ module RequestLogAnalyzer::Source
|
|
98
95
|
# Check if a file has a compressed extention in the filename.
|
99
96
|
# If recognized, return the command string used to decompress the file
|
100
97
|
def decompress_file?(filename)
|
101
|
-
nice_command =
|
98
|
+
nice_command = 'nice -n 5'
|
102
99
|
|
103
100
|
return "#{nice_command} gunzip -c -d #{filename}" if filename.match(/\.tar.gz$/) || filename.match(/\.tgz$/) || filename.match(/\.gz$/)
|
104
101
|
return "#{nice_command} bunzip2 -c -d #{filename}" if filename.match(/\.bz2$/)
|
105
102
|
return "#{nice_command} unzip -p #{filename}" if filename.match(/\.zip$/)
|
106
103
|
|
107
|
-
|
104
|
+
''
|
108
105
|
end
|
109
106
|
|
110
107
|
# Parses a log file. Creates an IO stream for the provided file, and sends it to parse_io for
|
@@ -117,7 +114,6 @@ module RequestLogAnalyzer::Source
|
|
117
114
|
# <tt>file</tt>:: The file that should be parsed.
|
118
115
|
# <tt>options</tt>:: A Hash of options that will be pased to parse_io.
|
119
116
|
def parse_file(file, options = {}, &block)
|
120
|
-
|
121
117
|
if File.directory?(file)
|
122
118
|
parse_files(Dir["#{ file }/*"], options, &block)
|
123
119
|
return
|
@@ -145,7 +141,6 @@ module RequestLogAnalyzer::Source
|
|
145
141
|
@source_changes_handler.call(:finished, @current_source) if @source_changes_handler
|
146
142
|
|
147
143
|
@current_source = nil
|
148
|
-
|
149
144
|
end
|
150
145
|
|
151
146
|
# Parses an IO stream. It will simply call parse_io. This function does not support progress updates
|
@@ -187,7 +182,7 @@ module RequestLogAnalyzer::Source
|
|
187
182
|
parse_line(line, &block)
|
188
183
|
end
|
189
184
|
|
190
|
-
warn(:unfinished_request_on_eof,
|
185
|
+
warn(:unfinished_request_on_eof, 'End of file reached, but last request was not completed!') unless @current_request.nil?
|
191
186
|
@current_lineno = nil
|
192
187
|
end
|
193
188
|
|
@@ -214,7 +209,7 @@ module RequestLogAnalyzer::Source
|
|
214
209
|
parse_line(line, &block)
|
215
210
|
end
|
216
211
|
|
217
|
-
warn(:unfinished_request_on_eof,
|
212
|
+
warn(:unfinished_request_on_eof, 'End of file reached, but last request was not completed!') unless @current_request.nil?
|
218
213
|
@current_lineno = nil
|
219
214
|
end
|
220
215
|
|
@@ -227,7 +222,7 @@ module RequestLogAnalyzer::Source
|
|
227
222
|
def parse_line(line, &block) # :yields: request
|
228
223
|
if request_data = file_format.parse_line(line) { |wt, message| warn(wt, message) }
|
229
224
|
@parsed_lines += 1
|
230
|
-
update_current_request(request_data.merge(:
|
225
|
+
update_current_request(request_data.merge(source: @current_source, lineno: @current_lineno), &block)
|
231
226
|
end
|
232
227
|
end
|
233
228
|
|
@@ -331,21 +326,19 @@ module RequestLogAnalyzer::Source
|
|
331
326
|
# - It will update the parsed_requests and skipped_requests variables accordingly
|
332
327
|
#
|
333
328
|
# <tt>request</tt>:: The parsed request instance (RequestLogAnalyzer::Request)
|
334
|
-
def handle_request(request, &
|
329
|
+
def handle_request(request, &_block) # :yields: :request, request
|
335
330
|
@parsed_requests += 1
|
336
331
|
request.validate
|
337
332
|
accepted = block_given? ? yield(request) : true
|
338
333
|
@skipped_requests += 1 unless accepted
|
339
334
|
end
|
340
335
|
|
341
|
-
|
342
336
|
# Checks whether a given line hash is an alternative header line according to the current file format.
|
343
337
|
# <tt>hash</tt>:: A hash of data that was parsed from the line.
|
344
338
|
def alternative_header_line?(hash)
|
345
339
|
hash[:line_definition].header == :alternative
|
346
340
|
end
|
347
341
|
|
348
|
-
|
349
342
|
# Checks whether a given line hash is a header line according to the current file format.
|
350
343
|
# <tt>hash</tt>:: A hash of data that was parsed from the line.
|
351
344
|
def header_line?(hash)
|