request-log-analyzer 1.13.1 → 1.13.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/bin/console +17 -0
- data/lib/cli/command_line_arguments.rb +29 -36
- data/lib/cli/database_console.rb +1 -3
- data/lib/cli/database_console_init.rb +11 -11
- data/lib/cli/progressbar.rb +30 -32
- data/lib/cli/tools.rb +20 -23
- data/lib/request_log_analyzer.rb +8 -8
- data/lib/request_log_analyzer/aggregator.rb +4 -7
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +10 -13
- data/lib/request_log_analyzer/aggregator/echo.rb +5 -7
- data/lib/request_log_analyzer/aggregator/summarizer.rb +15 -18
- data/lib/request_log_analyzer/class_level_inheritable_attributes.rb +23 -0
- data/lib/request_log_analyzer/controller.rb +36 -42
- data/lib/request_log_analyzer/database.rb +4 -6
- data/lib/request_log_analyzer/database/base.rb +39 -41
- data/lib/request_log_analyzer/database/connection.rb +8 -10
- data/lib/request_log_analyzer/database/request.rb +1 -3
- data/lib/request_log_analyzer/database/source.rb +0 -2
- data/lib/request_log_analyzer/database/warning.rb +4 -6
- data/lib/request_log_analyzer/file_format.rb +46 -49
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +15 -19
- data/lib/request_log_analyzer/file_format/apache.rb +42 -45
- data/lib/request_log_analyzer/file_format/delayed_job.rb +13 -15
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job3.rb +5 -8
- data/lib/request_log_analyzer/file_format/delayed_job4.rb +5 -8
- data/lib/request_log_analyzer/file_format/haproxy.rb +44 -48
- data/lib/request_log_analyzer/file_format/merb.rb +13 -17
- data/lib/request_log_analyzer/file_format/mysql.rb +21 -25
- data/lib/request_log_analyzer/file_format/nginx.rb +0 -2
- data/lib/request_log_analyzer/file_format/oink.rb +30 -31
- data/lib/request_log_analyzer/file_format/postgresql.rb +11 -15
- data/lib/request_log_analyzer/file_format/rack.rb +0 -2
- data/lib/request_log_analyzer/file_format/rails.rb +100 -104
- data/lib/request_log_analyzer/file_format/rails3.rb +19 -23
- data/lib/request_log_analyzer/file_format/rails_development.rb +0 -1
- data/lib/request_log_analyzer/file_format/w3c.rb +16 -18
- data/lib/request_log_analyzer/filter.rb +0 -2
- data/lib/request_log_analyzer/filter/anonymize.rb +4 -7
- data/lib/request_log_analyzer/filter/field.rb +3 -6
- data/lib/request_log_analyzer/filter/timespan.rb +2 -6
- data/lib/request_log_analyzer/line_definition.rb +16 -19
- data/lib/request_log_analyzer/log_processor.rb +10 -14
- data/lib/request_log_analyzer/mailer.rb +9 -12
- data/lib/request_log_analyzer/output.rb +12 -14
- data/lib/request_log_analyzer/output/fixed_width.rb +21 -28
- data/lib/request_log_analyzer/output/html.rb +11 -14
- data/lib/request_log_analyzer/request.rb +53 -33
- data/lib/request_log_analyzer/source.rb +2 -5
- data/lib/request_log_analyzer/source/log_parser.rb +9 -16
- data/lib/request_log_analyzer/tracker.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +4 -6
- data/lib/request_log_analyzer/tracker/frequency.rb +9 -11
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -11
- data/lib/request_log_analyzer/tracker/numeric_value.rb +40 -44
- data/lib/request_log_analyzer/tracker/timespan.rb +5 -8
- data/lib/request_log_analyzer/tracker/traffic.rb +8 -10
- data/lib/request_log_analyzer/version.rb +1 -1
- data/request-log-analyzer.gemspec +6 -6
- data/spec/integration/command_line_usage_spec.rb +33 -33
- data/spec/integration/mailer_spec.rb +181 -185
- data/spec/integration/munin_plugins_rails_spec.rb +20 -20
- data/spec/integration/scout_spec.rb +40 -41
- data/spec/lib/helpers.rb +8 -9
- data/spec/lib/macros.rb +2 -4
- data/spec/lib/matchers.rb +20 -25
- data/spec/lib/mocks.rb +10 -11
- data/spec/lib/testing_format.rb +8 -10
- data/spec/spec_helper.rb +5 -1
- data/spec/unit/aggregator/database_inserter_spec.rb +23 -23
- data/spec/unit/aggregator/summarizer_spec.rb +7 -7
- data/spec/unit/controller/controller_spec.rb +14 -14
- data/spec/unit/controller/log_processor_spec.rb +3 -3
- data/spec/unit/database/base_class_spec.rb +36 -37
- data/spec/unit/database/connection_spec.rb +10 -10
- data/spec/unit/database/database_spec.rb +11 -11
- data/spec/unit/file_format/amazon_s3_format_spec.rb +66 -62
- data/spec/unit/file_format/apache_format_spec.rb +57 -52
- data/spec/unit/file_format/common_regular_expressions_spec.rb +18 -21
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job2_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job3_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job4_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job_format_spec.rb +12 -12
- data/spec/unit/file_format/file_format_api_spec.rb +19 -19
- data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
- data/spec/unit/file_format/haproxy_format_spec.rb +53 -49
- data/spec/unit/file_format/inheritance_spec.rb +13 -0
- data/spec/unit/file_format/line_definition_spec.rb +35 -33
- data/spec/unit/file_format/merb_format_spec.rb +13 -11
- data/spec/unit/file_format/mysql_format_spec.rb +24 -24
- data/spec/unit/file_format/oink_format_spec.rb +29 -29
- data/spec/unit/file_format/postgresql_format_spec.rb +9 -9
- data/spec/unit/file_format/rack_format_spec.rb +36 -31
- data/spec/unit/file_format/rails3_format_spec.rb +46 -46
- data/spec/unit/file_format/rails_format_spec.rb +52 -53
- data/spec/unit/file_format/w3c_format_spec.rb +27 -24
- data/spec/unit/filter/anonymize_filter_spec.rb +7 -7
- data/spec/unit/filter/field_filter_spec.rb +26 -26
- data/spec/unit/filter/filter_spec.rb +4 -4
- data/spec/unit/filter/timespan_filter_spec.rb +22 -22
- data/spec/unit/mailer_spec.rb +21 -21
- data/spec/unit/request_spec.rb +29 -29
- data/spec/unit/source/log_parser_spec.rb +5 -5
- data/spec/unit/tracker/duration_tracker_spec.rb +23 -23
- data/spec/unit/tracker/frequency_tracker_spec.rb +29 -30
- data/spec/unit/tracker/hourly_spread_spec.rb +35 -35
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +71 -72
- data/spec/unit/tracker/timespan_tracker_spec.rb +31 -31
- data/spec/unit/tracker/tracker_api_spec.rb +43 -44
- data/spec/unit/tracker/traffic_tracker_spec.rb +7 -7
- metadata +38 -35
|
@@ -1,22 +1,19 @@
|
|
|
1
1
|
# coding: utf-8
|
|
2
2
|
module RequestLogAnalyzer::Output
|
|
3
|
-
|
|
4
3
|
# Fixed Width output class.
|
|
5
4
|
# Outputs a fixed width ASCII or UF8 report.
|
|
6
5
|
class FixedWidth < Base
|
|
7
|
-
|
|
8
6
|
# Mixin module. Will disable any colorizing.
|
|
9
7
|
module Monochrome
|
|
10
|
-
def colorize(text, *
|
|
8
|
+
def colorize(text, *_options)
|
|
11
9
|
text
|
|
12
10
|
end
|
|
13
11
|
end
|
|
14
12
|
|
|
15
13
|
# Colorize module
|
|
16
14
|
module Color
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
COLORS = { :black => 0, :blue => 4, :green => 2, :cyan => 6, :red => 1, :purple => 5, :brown => 3, :white => 7 }
|
|
15
|
+
STYLES = { normal: 0, bold: 1, underscore: 4, blink: 5, inverse: 7, concealed: 8 }
|
|
16
|
+
COLORS = { black: 0, blue: 4, green: 2, cyan: 6, red: 1, purple: 5, brown: 3, white: 7 }
|
|
20
17
|
|
|
21
18
|
# Colorize text
|
|
22
19
|
# <tt>text</tt> The text to colorize
|
|
@@ -28,36 +25,34 @@ module RequestLogAnalyzer::Output
|
|
|
28
25
|
#
|
|
29
26
|
# Returns ASCII colored string
|
|
30
27
|
def colorize(text, *options)
|
|
31
|
-
|
|
32
28
|
font_style = ''
|
|
33
29
|
foreground_color = '0'
|
|
34
30
|
background_color = ''
|
|
35
31
|
|
|
36
32
|
options.each do |option|
|
|
37
|
-
if option.
|
|
33
|
+
if option.is_a?(Symbol)
|
|
38
34
|
foreground_color = "3#{COLORS[option]}" if COLORS.include?(option)
|
|
39
35
|
font_style = "#{STYLES[option]};" if STYLES.include?(option)
|
|
40
|
-
elsif option.
|
|
36
|
+
elsif option.is_a?(Hash)
|
|
41
37
|
option.each do |key, value|
|
|
42
38
|
case key
|
|
43
|
-
when :color
|
|
44
|
-
when :background
|
|
45
|
-
when :on
|
|
46
|
-
when :style
|
|
39
|
+
when :color then foreground_color = "3#{COLORS[value]}" if COLORS.include?(value)
|
|
40
|
+
when :background then background_color = "4#{COLORS[value]};" if COLORS.include?(value)
|
|
41
|
+
when :on then background_color = "4#{COLORS[value]};" if COLORS.include?(value)
|
|
42
|
+
when :style then font_style = "#{STYLES[value]};" if STYLES.include?(value)
|
|
47
43
|
end
|
|
48
44
|
end
|
|
49
45
|
end
|
|
50
46
|
end
|
|
51
|
-
|
|
47
|
+
"\e[#{background_color}#{font_style}#{foreground_color}m#{text}\e[0m"
|
|
52
48
|
end
|
|
53
|
-
|
|
54
49
|
end
|
|
55
50
|
|
|
56
51
|
attr_reader :characters
|
|
57
52
|
|
|
58
53
|
CHARACTERS = {
|
|
59
|
-
:
|
|
60
|
-
:
|
|
54
|
+
ascii: { horizontal_line: '-', vertical_line: '|', block: '=' },
|
|
55
|
+
utf: { horizontal_line: '━', vertical_line: '┃', block: '░' }
|
|
61
56
|
}
|
|
62
57
|
|
|
63
58
|
# Initialize a report
|
|
@@ -82,7 +77,7 @@ module RequestLogAnalyzer::Output
|
|
|
82
77
|
@io << str
|
|
83
78
|
end
|
|
84
79
|
|
|
85
|
-
|
|
80
|
+
alias_method :<<, :print
|
|
86
81
|
|
|
87
82
|
# Write a string to the output object with a newline at the end.
|
|
88
83
|
# <tt>str</tt> The string to write.
|
|
@@ -116,8 +111,8 @@ module RequestLogAnalyzer::Output
|
|
|
116
111
|
|
|
117
112
|
# Generate a header for a report
|
|
118
113
|
def header
|
|
119
|
-
if io.
|
|
120
|
-
puts colorize(
|
|
114
|
+
if io.is_a?(File)
|
|
115
|
+
puts colorize('Request-log-analyzer summary report', :white, :bold)
|
|
121
116
|
line(:green)
|
|
122
117
|
puts "Version #{RequestLogAnalyzer::VERSION} - written by Willem van Bergen and Bart ten Brinke"
|
|
123
118
|
puts "Website: #{link('http://github.com/wvanbergen/request-log-analyzer')}"
|
|
@@ -127,7 +122,7 @@ module RequestLogAnalyzer::Output
|
|
|
127
122
|
# Generate a footer for a report
|
|
128
123
|
def footer
|
|
129
124
|
puts
|
|
130
|
-
puts
|
|
125
|
+
puts 'Need an expert to analyze your application?'
|
|
131
126
|
puts "Mail to #{link('contact@railsdoctors.com')} or visit us at #{link('http://railsdoctors.com')}."
|
|
132
127
|
line(:green)
|
|
133
128
|
puts "Thanks for using #{colorize('request-log-analyzer', :white, :bold)}!"
|
|
@@ -136,13 +131,12 @@ module RequestLogAnalyzer::Output
|
|
|
136
131
|
# Generate a report table and push it into the output object.
|
|
137
132
|
# <tt>*colums<tt> Columns hash
|
|
138
133
|
# <tt>&block</tt>: A block yeilding the rows.
|
|
139
|
-
def table(*columns, &
|
|
140
|
-
|
|
141
|
-
rows = Array.new
|
|
134
|
+
def table(*columns, &_block)
|
|
135
|
+
rows = []
|
|
142
136
|
yield(rows)
|
|
143
137
|
|
|
144
138
|
# determine maximum cell widths
|
|
145
|
-
max_cell_widths = rows.
|
|
139
|
+
max_cell_widths = rows.reduce(Array.new(columns.length, 0)) do |result, row|
|
|
146
140
|
lengths = row.map { |column| column.to_s.length }
|
|
147
141
|
result.each_with_index { |length, index| result[index] = ([length, lengths[index]].max rescue length) }
|
|
148
142
|
end
|
|
@@ -164,7 +158,7 @@ module RequestLogAnalyzer::Output
|
|
|
164
158
|
end
|
|
165
159
|
|
|
166
160
|
if column_widths.include?(nil)
|
|
167
|
-
width_left = options[:width] - ((columns.length - 1) * (style[:cell_separator] ? 3 : 1)) - column_widths.compact.
|
|
161
|
+
width_left = options[:width] - ((columns.length - 1) * (style[:cell_separator] ? 3 : 1)) - column_widths.compact.reduce(0) { |sum, col| sum + col }
|
|
168
162
|
column_widths[column_widths.index(nil)] = width_left
|
|
169
163
|
end
|
|
170
164
|
|
|
@@ -214,6 +208,5 @@ module RequestLogAnalyzer::Output
|
|
|
214
208
|
puts row_values.join(style[:cell_separator] ? " #{characters[:vertical_line]} " : ' ')
|
|
215
209
|
end
|
|
216
210
|
end
|
|
217
|
-
|
|
218
211
|
end
|
|
219
|
-
end
|
|
212
|
+
end
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
module RequestLogAnalyzer::Output
|
|
2
|
-
|
|
3
2
|
# HTML Output class. Generated a HTML-formatted report, including CSS.
|
|
4
3
|
class HTML < Base
|
|
5
|
-
|
|
6
4
|
# def initialize(io, options = {})
|
|
7
5
|
# super(io, options)
|
|
8
6
|
# end
|
|
@@ -20,7 +18,7 @@ module RequestLogAnalyzer::Output
|
|
|
20
18
|
@io << str
|
|
21
19
|
end
|
|
22
20
|
|
|
23
|
-
|
|
21
|
+
alias_method :<<, :print
|
|
24
22
|
|
|
25
23
|
# Put a string with newline
|
|
26
24
|
def puts(str = '')
|
|
@@ -34,7 +32,7 @@ module RequestLogAnalyzer::Output
|
|
|
34
32
|
|
|
35
33
|
# Render a single line
|
|
36
34
|
# <tt>*font</tt> The font.
|
|
37
|
-
def line(*
|
|
35
|
+
def line(*_font)
|
|
38
36
|
@io.puts(tag(:hr))
|
|
39
37
|
end
|
|
40
38
|
|
|
@@ -43,17 +41,17 @@ module RequestLogAnalyzer::Output
|
|
|
43
41
|
# <tt>url</tt> The url to link to.
|
|
44
42
|
def link(text, url = nil)
|
|
45
43
|
url = text if url.nil?
|
|
46
|
-
tag(:a, text, :
|
|
44
|
+
tag(:a, text, href: url)
|
|
47
45
|
end
|
|
48
46
|
|
|
49
47
|
# Generate a report table in HTML and push it into the output object.
|
|
50
48
|
# <tt>*colums<tt> Columns hash
|
|
51
49
|
# <tt>&block</tt>: A block yeilding the rows.
|
|
52
|
-
def table(*columns, &
|
|
53
|
-
rows =
|
|
50
|
+
def table(*columns, &_block)
|
|
51
|
+
rows = []
|
|
54
52
|
yield(rows)
|
|
55
53
|
|
|
56
|
-
@io << tag(:table,
|
|
54
|
+
@io << tag(:table, class: 'rla-report-table', cellspacing: 0) do |content|
|
|
57
55
|
if table_has_header?(columns)
|
|
58
56
|
content << tag(:tr) do
|
|
59
57
|
columns.map { |col| tag(:th, col[:title]) }.join("\n")
|
|
@@ -65,21 +63,20 @@ module RequestLogAnalyzer::Output
|
|
|
65
63
|
odd = !odd
|
|
66
64
|
content << tag(:tr) do
|
|
67
65
|
if odd
|
|
68
|
-
row.map { |cell| tag(:td, cell, :
|
|
66
|
+
row.map { |cell| tag(:td, cell, class: 'alt') }.join("\n")
|
|
69
67
|
else
|
|
70
68
|
row.map { |cell| tag(:td, cell) }.join("\n")
|
|
71
69
|
end
|
|
72
70
|
end
|
|
73
71
|
end
|
|
74
72
|
end
|
|
75
|
-
|
|
76
73
|
end
|
|
77
74
|
|
|
78
75
|
# Genrate HTML header and associated stylesheet
|
|
79
76
|
def header
|
|
80
77
|
@io.content_type = content_type if @io.respond_to?(:content_type)
|
|
81
78
|
|
|
82
|
-
@io <<
|
|
79
|
+
@io << '<html>'
|
|
83
80
|
@io << tag(:head) do |headers|
|
|
84
81
|
headers << tag(:title, 'Request-log-analyzer report')
|
|
85
82
|
headers << tag(:style, '
|
|
@@ -111,7 +108,7 @@ module RequestLogAnalyzer::Output
|
|
|
111
108
|
|
|
112
109
|
caption {
|
|
113
110
|
padding: 0 0 5px 0;
|
|
114
|
-
width: 700px;
|
|
111
|
+
width: 700px;
|
|
115
112
|
font: italic 11px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
|
116
113
|
text-align: right;
|
|
117
114
|
}
|
|
@@ -142,7 +139,7 @@ module RequestLogAnalyzer::Output
|
|
|
142
139
|
background: #F5FAFA;
|
|
143
140
|
color: #797268;
|
|
144
141
|
}
|
|
145
|
-
', :
|
|
142
|
+
', type: 'text/css')
|
|
146
143
|
end
|
|
147
144
|
@io << '<body>'
|
|
148
145
|
@io << tag(:h1, 'Request-log-analyzer summary report')
|
|
@@ -176,7 +173,7 @@ module RequestLogAnalyzer::Output
|
|
|
176
173
|
"<#{tag}#{attributes} />"
|
|
177
174
|
else
|
|
178
175
|
if content.class == Float
|
|
179
|
-
"<#{tag}#{attributes}><div class='color_bar' style=\"width:#{(content*200).floor}px;\"/></#{tag}>"
|
|
176
|
+
"<#{tag}#{attributes}><div class='color_bar' style=\"width:#{(content * 200).floor}px;\"/></#{tag}>"
|
|
180
177
|
else
|
|
181
178
|
"<#{tag}#{attributes}>#{content}</#{tag}>"
|
|
182
179
|
end
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
module RequestLogAnalyzer
|
|
2
|
-
|
|
3
2
|
# The Request class represents a parsed request from the log file.
|
|
4
3
|
# Instances are created by the LogParser and are passed to the different aggregators, so they
|
|
5
4
|
# can do their aggregating work.
|
|
@@ -8,9 +7,7 @@ module RequestLogAnalyzer
|
|
|
8
7
|
# Request#first(field_name) returns the first (only) value corresponding to the given field
|
|
9
8
|
# Request#every(field_name) returns all values corresponding to the given field name as array.
|
|
10
9
|
class Request
|
|
11
|
-
|
|
12
10
|
module Converters
|
|
13
|
-
|
|
14
11
|
# Default converter function, which converts the parsed strings to a native Ruby type
|
|
15
12
|
# using the type indication in the line definition. It will use a custom connverter
|
|
16
13
|
# method if one is available.
|
|
@@ -20,24 +17,47 @@ module RequestLogAnalyzer
|
|
|
20
17
|
send(custom_converter_method, value, capture_definition)
|
|
21
18
|
end
|
|
22
19
|
|
|
23
|
-
def convert_string(value,
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
20
|
+
def convert_string(value, _capture_definition)
|
|
21
|
+
value
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def convert_float(value, _capture_definition)
|
|
25
|
+
value.to_f
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def convert_decimal(value, _capture_definition)
|
|
29
|
+
value.to_f
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def convert_int(value, _capture_definition)
|
|
33
|
+
value.to_i
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def convert_integer(value, _capture_definition)
|
|
37
|
+
value.to_i
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def convert_sym(value, _capture_definition)
|
|
41
|
+
value.to_sym
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def convert_symbol(value, _capture_definition)
|
|
45
|
+
value.to_sym
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def convert_nillable_string(value, _definition)
|
|
49
|
+
value == '-' ? nil : value
|
|
50
|
+
end
|
|
51
|
+
|
|
32
52
|
# This function can be overridden to rewrite the path for better categorization in the
|
|
33
53
|
# reports.
|
|
34
|
-
def convert_path(value,
|
|
54
|
+
def convert_path(value, _definition)
|
|
35
55
|
value
|
|
36
56
|
end
|
|
37
57
|
|
|
38
58
|
# Converts :eval field, which should evaluate to a hash.
|
|
39
|
-
def convert_eval(value,
|
|
40
|
-
eval(sanitize_parameters(value)).
|
|
59
|
+
def convert_eval(value, _capture_definition)
|
|
60
|
+
eval(sanitize_parameters(value)).reduce({}) { |h, (k, v)| h[k.to_sym] = v; h }
|
|
41
61
|
rescue
|
|
42
62
|
nil
|
|
43
63
|
end
|
|
@@ -51,7 +71,7 @@ module RequestLogAnalyzer
|
|
|
51
71
|
# Slow default method to parse timestamps.
|
|
52
72
|
# Reimplement this function in a file format specific Request class
|
|
53
73
|
# to improve the timestamp parsing speed.
|
|
54
|
-
def convert_timestamp(value,
|
|
74
|
+
def convert_timestamp(value, _capture_definition)
|
|
55
75
|
DateTime.parse(value).strftime('%Y%m%d%H%M%S').to_i
|
|
56
76
|
end
|
|
57
77
|
|
|
@@ -59,13 +79,13 @@ module RequestLogAnalyzer
|
|
|
59
79
|
def convert_traffic(value, capture_definition)
|
|
60
80
|
case capture_definition[:unit]
|
|
61
81
|
when nil, :b, :B, :byte then value.to_i
|
|
62
|
-
when :GB, :G, :gigabyte then (value.to_f *
|
|
63
|
-
when :GiB, :gibibyte then (value.to_f * (2
|
|
64
|
-
when :MB, :M, :megabyte then (value.to_f *
|
|
65
|
-
when :MiB, :mebibyte then (value.to_f * (2
|
|
82
|
+
when :GB, :G, :gigabyte then (value.to_f * 1_000_000_000).round
|
|
83
|
+
when :GiB, :gibibyte then (value.to_f * (2**30)).round
|
|
84
|
+
when :MB, :M, :megabyte then (value.to_f * 1_000_000).round
|
|
85
|
+
when :MiB, :mebibyte then (value.to_f * (2**20)).round
|
|
66
86
|
when :KB, :K, :kilobyte, :kB then (value.to_f * 1000).round
|
|
67
|
-
when :KiB, :kibibyte then (value.to_f * (2
|
|
68
|
-
else
|
|
87
|
+
when :KiB, :kibibyte then (value.to_f * (2**10)).round
|
|
88
|
+
else fail 'Unknown traffic unit'
|
|
69
89
|
end
|
|
70
90
|
end
|
|
71
91
|
|
|
@@ -73,14 +93,14 @@ module RequestLogAnalyzer
|
|
|
73
93
|
def convert_duration(value, capture_definition)
|
|
74
94
|
case capture_definition[:unit]
|
|
75
95
|
when nil, :sec, :s then value.to_f
|
|
76
|
-
when :microsec, :musec then value.to_f /
|
|
96
|
+
when :microsec, :musec then value.to_f / 1_000_000.0
|
|
77
97
|
when :msec, :millisec then value.to_f / 1000.0
|
|
78
|
-
else
|
|
98
|
+
else fail 'Unknown duration unit'
|
|
79
99
|
end
|
|
80
100
|
end
|
|
81
|
-
|
|
101
|
+
|
|
82
102
|
# Convert an epoch to an integer
|
|
83
|
-
def convert_epoch(value,
|
|
103
|
+
def convert_epoch(value, _capture_definition)
|
|
84
104
|
Time.at(value.to_i).strftime('%Y%m%d%H%M%S').to_i
|
|
85
105
|
end
|
|
86
106
|
end
|
|
@@ -101,16 +121,16 @@ module RequestLogAnalyzer
|
|
|
101
121
|
# Creates a new request that was parsed from the log with the given FileFormat. The hashes
|
|
102
122
|
# that are passed to this function are added as lines to this request.
|
|
103
123
|
def self.create(file_format, *hashes)
|
|
104
|
-
request =
|
|
124
|
+
request = new(file_format)
|
|
105
125
|
hashes.flatten.each { |hash| request << hash }
|
|
106
|
-
|
|
126
|
+
request
|
|
107
127
|
end
|
|
108
128
|
|
|
109
129
|
# Adds another line to the request when it is parsed in the LogParser.
|
|
110
130
|
#
|
|
111
131
|
# The line should be provided as a hash with the attributes line_definition, :captures,
|
|
112
132
|
# :lineno and :source set. This function is called from LogParser.
|
|
113
|
-
def add_parsed_line
|
|
133
|
+
def add_parsed_line(parsed_line)
|
|
114
134
|
value_hash = parsed_line[:line_definition].convert_captured_values(parsed_line[:captures], self)
|
|
115
135
|
value_hash[:line_type] = parsed_line[:line_definition].name
|
|
116
136
|
value_hash[:lineno] = parsed_line[:lineno]
|
|
@@ -148,10 +168,10 @@ module RequestLogAnalyzer
|
|
|
148
168
|
# Checks whether the given line type was parsed from the log file for this request
|
|
149
169
|
def has_line_type?(line_type)
|
|
150
170
|
return true if @lines.length == 1 && @lines[0][:line_type] == line_type.to_sym
|
|
151
|
-
@lines.
|
|
171
|
+
@lines.find { |l| l[:line_type] == line_type.to_sym }
|
|
152
172
|
end
|
|
153
173
|
|
|
154
|
-
|
|
174
|
+
alias_method :=~, :has_line_type?
|
|
155
175
|
|
|
156
176
|
# Returns the value that was captured for the "field" of this request.
|
|
157
177
|
# This function will return the first value that was captured if the field
|
|
@@ -160,11 +180,11 @@ module RequestLogAnalyzer
|
|
|
160
180
|
@attributes[field]
|
|
161
181
|
end
|
|
162
182
|
|
|
163
|
-
|
|
183
|
+
alias_method :[], :first
|
|
164
184
|
|
|
165
185
|
# Returns an array of all the "field" values that were captured for this request
|
|
166
186
|
def every(field)
|
|
167
|
-
@lines.
|
|
187
|
+
@lines.reduce([]) { |result, fields| result << fields[field] if fields.key?(field); result }
|
|
168
188
|
end
|
|
169
189
|
|
|
170
190
|
# Returns true if this request does not yet contain any parsed lines. This should only occur
|
|
@@ -7,13 +7,11 @@
|
|
|
7
7
|
# - The base class for all sources is RequestLogAnalyzer::Source::Base. All source classes should inherit from this class.
|
|
8
8
|
# - Currently, RequestLogAnalyzer::Source::LogParser is the only implemented source.
|
|
9
9
|
module RequestLogAnalyzer::Source
|
|
10
|
-
|
|
11
10
|
# The base Source class. All other sources should inherit from this class.
|
|
12
11
|
#
|
|
13
12
|
# A source implememtation should at least implement the each_request method, which should yield
|
|
14
13
|
# RequestLogAnalyzer::Request instances that will be fed through the pipleine.
|
|
15
14
|
class Base
|
|
16
|
-
|
|
17
15
|
# A hash of options
|
|
18
16
|
attr_reader :options
|
|
19
17
|
|
|
@@ -52,15 +50,14 @@ module RequestLogAnalyzer::Source
|
|
|
52
50
|
# This function is called to actually produce the requests that will be send into the pipeline.
|
|
53
51
|
# The implementation should yield instances of RequestLogAnalyzer::Request.
|
|
54
52
|
# <tt>options</tt>:: A Hash of options that can be used in the implementation.
|
|
55
|
-
def each_request(
|
|
56
|
-
|
|
53
|
+
def each_request(_options = {}, &_block) # :yields: request
|
|
54
|
+
true
|
|
57
55
|
end
|
|
58
56
|
|
|
59
57
|
# This function is called after RequestLogAnalyzer::Source::Base#each_request finished. Any code to
|
|
60
58
|
# wrap up, free resources, etc. can be put in this method.
|
|
61
59
|
def finalize
|
|
62
60
|
end
|
|
63
|
-
|
|
64
61
|
end
|
|
65
62
|
end
|
|
66
63
|
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
module RequestLogAnalyzer::Source
|
|
2
|
-
|
|
3
2
|
# The LogParser class reads log data from a given source and uses a file format definition
|
|
4
3
|
# to parse all relevent information about requests from the file. A FileFormat module should
|
|
5
4
|
# be provided that contains the definitions of the lines that occur in the log data.
|
|
@@ -10,7 +9,6 @@ module RequestLogAnalyzer::Source
|
|
|
10
9
|
# parser. It will emit warnings when this occurs. LogParser supports multiple parse strategies
|
|
11
10
|
# that deal differently with this problem.
|
|
12
11
|
class LogParser < Base
|
|
13
|
-
|
|
14
12
|
include Enumerable
|
|
15
13
|
|
|
16
14
|
# The maximum number of bytes to read from a line.
|
|
@@ -51,7 +49,7 @@ module RequestLogAnalyzer::Source
|
|
|
51
49
|
|
|
52
50
|
@options[:parse_strategy] ||= DEFAULT_PARSE_STRATEGY
|
|
53
51
|
unless PARSE_STRATEGIES.include?(@options[:parse_strategy])
|
|
54
|
-
|
|
52
|
+
fail "Unknown parse strategy: #{@options[@parse_strategy]}"
|
|
55
53
|
end
|
|
56
54
|
end
|
|
57
55
|
|
|
@@ -68,11 +66,10 @@ module RequestLogAnalyzer::Source
|
|
|
68
66
|
# that will be yielded. The actual parsing occurs in the parse_io method.
|
|
69
67
|
# <tt>options</tt>:: A Hash of options that will be pased to parse_io.
|
|
70
68
|
def each_request(options = {}, &block) # :yields: :request, request
|
|
71
|
-
|
|
72
69
|
case @source_files
|
|
73
70
|
when IO
|
|
74
71
|
if @source_files == $stdin
|
|
75
|
-
puts
|
|
72
|
+
puts 'Parsing from the standard input. Press CTRL+C to finish.' # FIXME: not here
|
|
76
73
|
end
|
|
77
74
|
parse_stream(@source_files, options, &block)
|
|
78
75
|
when String
|
|
@@ -80,7 +77,7 @@ module RequestLogAnalyzer::Source
|
|
|
80
77
|
when Array
|
|
81
78
|
parse_files(@source_files, options, &block)
|
|
82
79
|
else
|
|
83
|
-
|
|
80
|
+
fail 'Unknown source provided'
|
|
84
81
|
end
|
|
85
82
|
end
|
|
86
83
|
|
|
@@ -98,13 +95,13 @@ module RequestLogAnalyzer::Source
|
|
|
98
95
|
# Check if a file has a compressed extention in the filename.
|
|
99
96
|
# If recognized, return the command string used to decompress the file
|
|
100
97
|
def decompress_file?(filename)
|
|
101
|
-
nice_command =
|
|
98
|
+
nice_command = 'nice -n 5'
|
|
102
99
|
|
|
103
100
|
return "#{nice_command} gunzip -c -d #{filename}" if filename.match(/\.tar.gz$/) || filename.match(/\.tgz$/) || filename.match(/\.gz$/)
|
|
104
101
|
return "#{nice_command} bunzip2 -c -d #{filename}" if filename.match(/\.bz2$/)
|
|
105
102
|
return "#{nice_command} unzip -p #{filename}" if filename.match(/\.zip$/)
|
|
106
103
|
|
|
107
|
-
|
|
104
|
+
''
|
|
108
105
|
end
|
|
109
106
|
|
|
110
107
|
# Parses a log file. Creates an IO stream for the provided file, and sends it to parse_io for
|
|
@@ -117,7 +114,6 @@ module RequestLogAnalyzer::Source
|
|
|
117
114
|
# <tt>file</tt>:: The file that should be parsed.
|
|
118
115
|
# <tt>options</tt>:: A Hash of options that will be pased to parse_io.
|
|
119
116
|
def parse_file(file, options = {}, &block)
|
|
120
|
-
|
|
121
117
|
if File.directory?(file)
|
|
122
118
|
parse_files(Dir["#{ file }/*"], options, &block)
|
|
123
119
|
return
|
|
@@ -145,7 +141,6 @@ module RequestLogAnalyzer::Source
|
|
|
145
141
|
@source_changes_handler.call(:finished, @current_source) if @source_changes_handler
|
|
146
142
|
|
|
147
143
|
@current_source = nil
|
|
148
|
-
|
|
149
144
|
end
|
|
150
145
|
|
|
151
146
|
# Parses an IO stream. It will simply call parse_io. This function does not support progress updates
|
|
@@ -187,7 +182,7 @@ module RequestLogAnalyzer::Source
|
|
|
187
182
|
parse_line(line, &block)
|
|
188
183
|
end
|
|
189
184
|
|
|
190
|
-
warn(:unfinished_request_on_eof,
|
|
185
|
+
warn(:unfinished_request_on_eof, 'End of file reached, but last request was not completed!') unless @current_request.nil?
|
|
191
186
|
@current_lineno = nil
|
|
192
187
|
end
|
|
193
188
|
|
|
@@ -214,7 +209,7 @@ module RequestLogAnalyzer::Source
|
|
|
214
209
|
parse_line(line, &block)
|
|
215
210
|
end
|
|
216
211
|
|
|
217
|
-
warn(:unfinished_request_on_eof,
|
|
212
|
+
warn(:unfinished_request_on_eof, 'End of file reached, but last request was not completed!') unless @current_request.nil?
|
|
218
213
|
@current_lineno = nil
|
|
219
214
|
end
|
|
220
215
|
|
|
@@ -227,7 +222,7 @@ module RequestLogAnalyzer::Source
|
|
|
227
222
|
def parse_line(line, &block) # :yields: request
|
|
228
223
|
if request_data = file_format.parse_line(line) { |wt, message| warn(wt, message) }
|
|
229
224
|
@parsed_lines += 1
|
|
230
|
-
update_current_request(request_data.merge(:
|
|
225
|
+
update_current_request(request_data.merge(source: @current_source, lineno: @current_lineno), &block)
|
|
231
226
|
end
|
|
232
227
|
end
|
|
233
228
|
|
|
@@ -331,21 +326,19 @@ module RequestLogAnalyzer::Source
|
|
|
331
326
|
# - It will update the parsed_requests and skipped_requests variables accordingly
|
|
332
327
|
#
|
|
333
328
|
# <tt>request</tt>:: The parsed request instance (RequestLogAnalyzer::Request)
|
|
334
|
-
def handle_request(request, &
|
|
329
|
+
def handle_request(request, &_block) # :yields: :request, request
|
|
335
330
|
@parsed_requests += 1
|
|
336
331
|
request.validate
|
|
337
332
|
accepted = block_given? ? yield(request) : true
|
|
338
333
|
@skipped_requests += 1 unless accepted
|
|
339
334
|
end
|
|
340
335
|
|
|
341
|
-
|
|
342
336
|
# Checks whether a given line hash is an alternative header line according to the current file format.
|
|
343
337
|
# <tt>hash</tt>:: A hash of data that was parsed from the line.
|
|
344
338
|
def alternative_header_line?(hash)
|
|
345
339
|
hash[:line_definition].header == :alternative
|
|
346
340
|
end
|
|
347
341
|
|
|
348
|
-
|
|
349
342
|
# Checks whether a given line hash is a header line according to the current file format.
|
|
350
343
|
# <tt>hash</tt>:: A hash of data that was parsed from the line.
|
|
351
344
|
def header_line?(hash)
|