request-log-analyzer 1.13.1 → 1.13.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/bin/console +17 -0
- data/lib/cli/command_line_arguments.rb +29 -36
- data/lib/cli/database_console.rb +1 -3
- data/lib/cli/database_console_init.rb +11 -11
- data/lib/cli/progressbar.rb +30 -32
- data/lib/cli/tools.rb +20 -23
- data/lib/request_log_analyzer.rb +8 -8
- data/lib/request_log_analyzer/aggregator.rb +4 -7
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +10 -13
- data/lib/request_log_analyzer/aggregator/echo.rb +5 -7
- data/lib/request_log_analyzer/aggregator/summarizer.rb +15 -18
- data/lib/request_log_analyzer/class_level_inheritable_attributes.rb +23 -0
- data/lib/request_log_analyzer/controller.rb +36 -42
- data/lib/request_log_analyzer/database.rb +4 -6
- data/lib/request_log_analyzer/database/base.rb +39 -41
- data/lib/request_log_analyzer/database/connection.rb +8 -10
- data/lib/request_log_analyzer/database/request.rb +1 -3
- data/lib/request_log_analyzer/database/source.rb +0 -2
- data/lib/request_log_analyzer/database/warning.rb +4 -6
- data/lib/request_log_analyzer/file_format.rb +46 -49
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +15 -19
- data/lib/request_log_analyzer/file_format/apache.rb +42 -45
- data/lib/request_log_analyzer/file_format/delayed_job.rb +13 -15
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job3.rb +5 -8
- data/lib/request_log_analyzer/file_format/delayed_job4.rb +5 -8
- data/lib/request_log_analyzer/file_format/haproxy.rb +44 -48
- data/lib/request_log_analyzer/file_format/merb.rb +13 -17
- data/lib/request_log_analyzer/file_format/mysql.rb +21 -25
- data/lib/request_log_analyzer/file_format/nginx.rb +0 -2
- data/lib/request_log_analyzer/file_format/oink.rb +30 -31
- data/lib/request_log_analyzer/file_format/postgresql.rb +11 -15
- data/lib/request_log_analyzer/file_format/rack.rb +0 -2
- data/lib/request_log_analyzer/file_format/rails.rb +100 -104
- data/lib/request_log_analyzer/file_format/rails3.rb +19 -23
- data/lib/request_log_analyzer/file_format/rails_development.rb +0 -1
- data/lib/request_log_analyzer/file_format/w3c.rb +16 -18
- data/lib/request_log_analyzer/filter.rb +0 -2
- data/lib/request_log_analyzer/filter/anonymize.rb +4 -7
- data/lib/request_log_analyzer/filter/field.rb +3 -6
- data/lib/request_log_analyzer/filter/timespan.rb +2 -6
- data/lib/request_log_analyzer/line_definition.rb +16 -19
- data/lib/request_log_analyzer/log_processor.rb +10 -14
- data/lib/request_log_analyzer/mailer.rb +9 -12
- data/lib/request_log_analyzer/output.rb +12 -14
- data/lib/request_log_analyzer/output/fixed_width.rb +21 -28
- data/lib/request_log_analyzer/output/html.rb +11 -14
- data/lib/request_log_analyzer/request.rb +53 -33
- data/lib/request_log_analyzer/source.rb +2 -5
- data/lib/request_log_analyzer/source/log_parser.rb +9 -16
- data/lib/request_log_analyzer/tracker.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +4 -6
- data/lib/request_log_analyzer/tracker/frequency.rb +9 -11
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -11
- data/lib/request_log_analyzer/tracker/numeric_value.rb +40 -44
- data/lib/request_log_analyzer/tracker/timespan.rb +5 -8
- data/lib/request_log_analyzer/tracker/traffic.rb +8 -10
- data/lib/request_log_analyzer/version.rb +1 -1
- data/request-log-analyzer.gemspec +6 -6
- data/spec/integration/command_line_usage_spec.rb +33 -33
- data/spec/integration/mailer_spec.rb +181 -185
- data/spec/integration/munin_plugins_rails_spec.rb +20 -20
- data/spec/integration/scout_spec.rb +40 -41
- data/spec/lib/helpers.rb +8 -9
- data/spec/lib/macros.rb +2 -4
- data/spec/lib/matchers.rb +20 -25
- data/spec/lib/mocks.rb +10 -11
- data/spec/lib/testing_format.rb +8 -10
- data/spec/spec_helper.rb +5 -1
- data/spec/unit/aggregator/database_inserter_spec.rb +23 -23
- data/spec/unit/aggregator/summarizer_spec.rb +7 -7
- data/spec/unit/controller/controller_spec.rb +14 -14
- data/spec/unit/controller/log_processor_spec.rb +3 -3
- data/spec/unit/database/base_class_spec.rb +36 -37
- data/spec/unit/database/connection_spec.rb +10 -10
- data/spec/unit/database/database_spec.rb +11 -11
- data/spec/unit/file_format/amazon_s3_format_spec.rb +66 -62
- data/spec/unit/file_format/apache_format_spec.rb +57 -52
- data/spec/unit/file_format/common_regular_expressions_spec.rb +18 -21
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job2_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job3_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job4_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job_format_spec.rb +12 -12
- data/spec/unit/file_format/file_format_api_spec.rb +19 -19
- data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
- data/spec/unit/file_format/haproxy_format_spec.rb +53 -49
- data/spec/unit/file_format/inheritance_spec.rb +13 -0
- data/spec/unit/file_format/line_definition_spec.rb +35 -33
- data/spec/unit/file_format/merb_format_spec.rb +13 -11
- data/spec/unit/file_format/mysql_format_spec.rb +24 -24
- data/spec/unit/file_format/oink_format_spec.rb +29 -29
- data/spec/unit/file_format/postgresql_format_spec.rb +9 -9
- data/spec/unit/file_format/rack_format_spec.rb +36 -31
- data/spec/unit/file_format/rails3_format_spec.rb +46 -46
- data/spec/unit/file_format/rails_format_spec.rb +52 -53
- data/spec/unit/file_format/w3c_format_spec.rb +27 -24
- data/spec/unit/filter/anonymize_filter_spec.rb +7 -7
- data/spec/unit/filter/field_filter_spec.rb +26 -26
- data/spec/unit/filter/filter_spec.rb +4 -4
- data/spec/unit/filter/timespan_filter_spec.rb +22 -22
- data/spec/unit/mailer_spec.rb +21 -21
- data/spec/unit/request_spec.rb +29 -29
- data/spec/unit/source/log_parser_spec.rb +5 -5
- data/spec/unit/tracker/duration_tracker_spec.rb +23 -23
- data/spec/unit/tracker/frequency_tracker_spec.rb +29 -30
- data/spec/unit/tracker/hourly_spread_spec.rb +35 -35
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +71 -72
- data/spec/unit/tracker/timespan_tracker_spec.rb +31 -31
- data/spec/unit/tracker/tracker_api_spec.rb +43 -44
- data/spec/unit/tracker/traffic_tracker_spec.rb +7 -7
- metadata +38 -35
data/lib/cli/tools.rb
CHANGED
|
@@ -6,27 +6,24 @@ module CommandLine
|
|
|
6
6
|
# If it is not possible to to so, it returns the default_width.
|
|
7
7
|
# <tt>default_width</tt> Defaults to 81
|
|
8
8
|
def terminal_width(default_width = 81, out = STDOUT)
|
|
9
|
-
|
|
9
|
+
tiocgwinsz = 0x5413
|
|
10
|
+
data = [0, 0, 0, 0].pack('SSSS')
|
|
11
|
+
if !RUBY_PLATFORM.include?('java') && out.ioctl(tiocgwinsz, data) >= 0 # JRuby crashes on ioctl
|
|
12
|
+
_, cols, _, _ = data.unpack('SSSS')
|
|
13
|
+
fail unless cols > 0
|
|
14
|
+
cols
|
|
15
|
+
else
|
|
16
|
+
fail
|
|
17
|
+
end
|
|
18
|
+
rescue
|
|
10
19
|
begin
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
raise unless cols > 0
|
|
16
|
-
cols
|
|
17
|
-
else
|
|
18
|
-
raise
|
|
20
|
+
IO.popen('stty -a 2>&1') do |pipe|
|
|
21
|
+
column_line = pipe.find { |line| /(\d+) columns/ =~ line }
|
|
22
|
+
fail unless column_line
|
|
23
|
+
Regexp.last_match[1].to_i
|
|
19
24
|
end
|
|
20
25
|
rescue
|
|
21
|
-
|
|
22
|
-
IO.popen('stty -a 2>&1') do |pipe|
|
|
23
|
-
column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
|
|
24
|
-
raise unless column_line
|
|
25
|
-
$1.to_i
|
|
26
|
-
end
|
|
27
|
-
rescue
|
|
28
|
-
default_width
|
|
29
|
-
end
|
|
26
|
+
default_width
|
|
30
27
|
end
|
|
31
28
|
end
|
|
32
29
|
|
|
@@ -40,14 +37,14 @@ module CommandLine
|
|
|
40
37
|
if File.directory?('./lib/tasks/')
|
|
41
38
|
task_file = File.expand_path('../../tasks/request_log_analyzer.rake', File.dirname(__FILE__))
|
|
42
39
|
FileUtils.copy(task_file, './lib/tasks/request_log_analyze.rake')
|
|
43
|
-
puts
|
|
44
|
-
puts
|
|
40
|
+
puts 'Installed rake tasks.'
|
|
41
|
+
puts 'To use, run: rake rla:report'
|
|
45
42
|
else
|
|
46
|
-
puts
|
|
47
|
-
puts
|
|
43
|
+
puts 'Cannot find /lib/tasks folder. Are you in your Rails directory?'
|
|
44
|
+
puts 'Installation aborted.'
|
|
48
45
|
end
|
|
49
46
|
else
|
|
50
|
-
|
|
47
|
+
fail "Cannot perform this install type! (#{install_type})"
|
|
51
48
|
end
|
|
52
49
|
end
|
|
53
50
|
end
|
data/lib/request_log_analyzer.rb
CHANGED
|
@@ -1,37 +1,37 @@
|
|
|
1
1
|
require 'date'
|
|
2
2
|
|
|
3
3
|
# RequestLogAnalyzer is the base namespace in which all functionality of RequestLogAnalyzer is implemented.
|
|
4
|
-
# This module
|
|
4
|
+
# This module itself contains some functions to help with class and source file loading. The actual
|
|
5
5
|
# application startup code resides in the {RequestLogAnalyzer::Controller} class.
|
|
6
6
|
#
|
|
7
7
|
# The {RequestLogAnalyzer::VERSION} constant can be used to determine what version of request-log-analyzer
|
|
8
8
|
# is running.
|
|
9
9
|
module RequestLogAnalyzer
|
|
10
|
-
|
|
11
|
-
#
|
|
12
|
-
# (<tt>request_log_analyzer/controller</tt>). This function can be used to load the file (using
|
|
10
|
+
# Convert a string/symbol in camel case ({RequestLogAnalyzer::Controller}) to underscores
|
|
11
|
+
# (<tt>request_log_analyzer/controller</tt>). This function can be used to load the file (using
|
|
13
12
|
# <tt>require</tt>) in which the given constant is defined.
|
|
14
13
|
#
|
|
15
14
|
# @param [#to_s] str The string-like to convert in the following format: <tt>ModuleName::ClassName</tt>.
|
|
16
15
|
# @return [String] The input string converted to underscore form.
|
|
17
16
|
def self.to_underscore(str)
|
|
18
|
-
str.to_s.gsub(/::/, '/').gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').gsub(/([a-z\d])([A-Z])/,'\1_\2').tr(
|
|
17
|
+
str.to_s.gsub(/::/, '/').gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').gsub(/([a-z\d])([A-Z])/, '\1_\2').tr('-', '_').downcase
|
|
19
18
|
end
|
|
20
19
|
|
|
21
20
|
# Convert a string/symbol in underscores (<tt>request_log_analyzer/controller</tt>) to camelcase
|
|
22
|
-
# ({RequestLogAnalyzer::Controller}). This can be used to find the class that is defined in a given
|
|
21
|
+
# ({RequestLogAnalyzer::Controller}). This can be used to find the class that is defined in a given
|
|
23
22
|
# filename.
|
|
24
23
|
#
|
|
25
24
|
# @param [#to_s] str The string-like to convert in the f`ollowing format: <tt>module_name/class_name</tt>.
|
|
26
|
-
# @return [String] The input string converted to camelcase form.
|
|
25
|
+
# @return [String] The input string converted to camelcase form.
|
|
27
26
|
def self.to_camelcase(str)
|
|
28
|
-
str.to_s.gsub(/\/(.?)/) {
|
|
27
|
+
str.to_s.gsub(/\/(.?)/) { '::' + Regexp.last_match[1].upcase }.gsub(/(^|_)(.)/) { Regexp.last_match[2].upcase }
|
|
29
28
|
end
|
|
30
29
|
end
|
|
31
30
|
|
|
32
31
|
require 'request_log_analyzer/version'
|
|
33
32
|
require 'request_log_analyzer/controller'
|
|
34
33
|
require 'request_log_analyzer/aggregator'
|
|
34
|
+
require 'request_log_analyzer/class_level_inheritable_attributes'
|
|
35
35
|
require 'request_log_analyzer/file_format'
|
|
36
36
|
require 'request_log_analyzer/filter'
|
|
37
37
|
require 'request_log_analyzer/line_definition'
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
module RequestLogAnalyzer::Aggregator
|
|
2
|
-
|
|
3
2
|
# The base class of an aggregator. This class provides the interface to which
|
|
4
3
|
# every aggregator should comply (by simply subclassing this class).
|
|
5
4
|
class Base
|
|
6
|
-
|
|
7
5
|
attr_reader :options, :source
|
|
8
6
|
|
|
9
7
|
# Intializes a new RequestLogAnalyzer::Aggregator::Base instance
|
|
@@ -20,7 +18,7 @@ module RequestLogAnalyzer::Aggregator
|
|
|
20
18
|
|
|
21
19
|
# The aggregate function is called for every request.
|
|
22
20
|
# Implement the aggregating functionality in this method
|
|
23
|
-
def aggregate(
|
|
21
|
+
def aggregate(_request)
|
|
24
22
|
end
|
|
25
23
|
|
|
26
24
|
# The finalize function is called after all sources are parsed and no more
|
|
@@ -29,18 +27,17 @@ module RequestLogAnalyzer::Aggregator
|
|
|
29
27
|
end
|
|
30
28
|
|
|
31
29
|
# The warning method is called if the parser eits a warning.
|
|
32
|
-
def warning(
|
|
30
|
+
def warning(_type, _message, _lineno)
|
|
33
31
|
end
|
|
34
32
|
|
|
35
33
|
# The report function is called at the end. Implement any result reporting
|
|
36
34
|
# in this function.
|
|
37
|
-
def report(
|
|
35
|
+
def report(_output)
|
|
38
36
|
end
|
|
39
37
|
|
|
40
38
|
# The source_change function gets called when handling a source is started or finished.
|
|
41
|
-
def source_change(
|
|
39
|
+
def source_change(_change, _filename)
|
|
42
40
|
end
|
|
43
|
-
|
|
44
41
|
end
|
|
45
42
|
end
|
|
46
43
|
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
module RequestLogAnalyzer::Aggregator
|
|
2
|
-
|
|
3
2
|
# The database aggregator will create an SQLite3 database with all parsed request information.
|
|
4
3
|
#
|
|
5
4
|
# The prepare method will create a database schema according to the file format definitions.
|
|
@@ -12,14 +11,13 @@ module RequestLogAnalyzer::Aggregator
|
|
|
12
11
|
# the request record, and a field for every parsed value. Finally, a warnings table will be
|
|
13
12
|
# created to log all parse warnings.
|
|
14
13
|
class DatabaseInserter < Base
|
|
15
|
-
|
|
16
14
|
attr_reader :request_count, :sources, :database
|
|
17
15
|
|
|
18
16
|
# Establishes a connection to the database and creates the necessary database schema for the
|
|
19
17
|
# current file format
|
|
20
18
|
def prepare
|
|
21
19
|
require 'request_log_analyzer/database'
|
|
22
|
-
|
|
20
|
+
|
|
23
21
|
@sources = {}
|
|
24
22
|
@database = RequestLogAnalyzer::Database.new(options[:database])
|
|
25
23
|
@database.file_format = source.file_format
|
|
@@ -32,14 +30,14 @@ module RequestLogAnalyzer::Aggregator
|
|
|
32
30
|
# This will create a record in the requests table and create a record for every line that has been parsed,
|
|
33
31
|
# in which the captured values will be stored.
|
|
34
32
|
def aggregate(request)
|
|
35
|
-
@request_object = RequestLogAnalyzer::Database::Request.new(:
|
|
33
|
+
@request_object = RequestLogAnalyzer::Database::Request.new(first_lineno: request.first_lineno, last_lineno: request.last_lineno)
|
|
36
34
|
request.lines.each do |line|
|
|
37
|
-
class_columns = database.get_class(line[:line_type]).column_names.reject { |column|
|
|
38
|
-
attributes = Hash[*line.select { |(key, _)| class_columns.include?(key.to_s)}.flatten]
|
|
39
|
-
|
|
35
|
+
class_columns = database.get_class(line[:line_type]).column_names.reject { |column| %w(id source_id request_id).include?(column) }
|
|
36
|
+
attributes = Hash[*line.select { |(key, _)| class_columns.include?(key.to_s) }.flatten]
|
|
37
|
+
|
|
40
38
|
# Fix encoding patch for 1.9.2
|
|
41
|
-
attributes.each do |k,v|
|
|
42
|
-
attributes[k] = v.force_encoding(
|
|
39
|
+
attributes.each do |k, v|
|
|
40
|
+
attributes[k] = v.force_encoding('UTF-8') if v.is_a?(String)
|
|
43
41
|
end
|
|
44
42
|
|
|
45
43
|
@request_object.send("#{line[:line_type]}_lines").build(attributes)
|
|
@@ -58,7 +56,7 @@ module RequestLogAnalyzer::Aggregator
|
|
|
58
56
|
|
|
59
57
|
# Records w warining in the warnings table.
|
|
60
58
|
def warning(type, message, lineno)
|
|
61
|
-
RequestLogAnalyzer::Database::Warning.create!(:
|
|
59
|
+
RequestLogAnalyzer::Database::Warning.create!(warning_type: type.to_s, message: message, lineno: lineno)
|
|
62
60
|
end
|
|
63
61
|
|
|
64
62
|
# Records source changes in the sources table
|
|
@@ -66,9 +64,9 @@ module RequestLogAnalyzer::Aggregator
|
|
|
66
64
|
if File.exist?(filename)
|
|
67
65
|
case change
|
|
68
66
|
when :started
|
|
69
|
-
@sources[filename] = RequestLogAnalyzer::Database::Source.create!(:
|
|
67
|
+
@sources[filename] = RequestLogAnalyzer::Database::Source.create!(filename: filename)
|
|
70
68
|
when :finished
|
|
71
|
-
@sources[filename].update_attributes!(:
|
|
69
|
+
@sources[filename].update_attributes!(filesize: File.size(filename), mtime: File.mtime(filename))
|
|
72
70
|
end
|
|
73
71
|
end
|
|
74
72
|
end
|
|
@@ -84,6 +82,5 @@ module RequestLogAnalyzer::Aggregator
|
|
|
84
82
|
output << output.colorize(" $ request-log-analyzer console -d #{options[:database]}\n", :bold)
|
|
85
83
|
output << "\n"
|
|
86
84
|
end
|
|
87
|
-
|
|
88
85
|
end
|
|
89
86
|
end
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
module RequestLogAnalyzer::Aggregator
|
|
2
|
-
|
|
3
2
|
# Echo Aggregator. Writes everything to the screen when it is passed to this aggregator
|
|
4
3
|
class Echo < Base
|
|
5
|
-
|
|
6
4
|
attr_accessor :warnings
|
|
7
5
|
|
|
8
6
|
def prepare
|
|
@@ -11,8 +9,9 @@ module RequestLogAnalyzer::Aggregator
|
|
|
11
9
|
|
|
12
10
|
# Display every parsed line immediately to the terminal
|
|
13
11
|
def aggregate(request)
|
|
14
|
-
puts "\nRequest: \n" + request.lines.map
|
|
15
|
-
"\t#{l[:lineno]}:#{l[:line_type]}: #{l.reject { |(k,_)| [:lineno, :line_type].include?(k) }.inspect}"
|
|
12
|
+
puts "\nRequest: \n" + request.lines.map do |l|
|
|
13
|
+
"\t#{l[:lineno]}:#{l[:line_type]}: #{l.reject { |(k, _)| [:lineno, :line_type].include?(k) }.inspect}"
|
|
14
|
+
end.join("\n")
|
|
16
15
|
end
|
|
17
16
|
|
|
18
17
|
# Capture all warnings during parsing
|
|
@@ -22,9 +21,8 @@ module RequestLogAnalyzer::Aggregator
|
|
|
22
21
|
|
|
23
22
|
# Display every warning in the report when finished parsing
|
|
24
23
|
def report(output)
|
|
25
|
-
output.title(
|
|
24
|
+
output.title('Warnings during parsing')
|
|
26
25
|
@warnings.each { |w| output.puts(w) }
|
|
27
26
|
end
|
|
28
|
-
|
|
29
27
|
end
|
|
30
|
-
end
|
|
28
|
+
end
|
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
module RequestLogAnalyzer::Aggregator
|
|
2
|
-
|
|
3
2
|
class Summarizer < Base
|
|
4
|
-
|
|
5
3
|
class Definer
|
|
6
|
-
|
|
7
4
|
attr_reader :trackers
|
|
8
5
|
|
|
9
6
|
# Initialize tracker array
|
|
@@ -31,8 +28,8 @@ module RequestLogAnalyzer::Aggregator
|
|
|
31
28
|
# <tt>tracker_class</tt> The class to include
|
|
32
29
|
# <tt>optiont</tt> The options to pass to the trackers.
|
|
33
30
|
def track(tracker_klass, value_field = {}, other_options = {})
|
|
34
|
-
options = value_field.
|
|
35
|
-
tracker_klass = RequestLogAnalyzer::Tracker.const_get(RequestLogAnalyzer.to_camelcase(tracker_klass)) if tracker_klass.
|
|
31
|
+
options = value_field.is_a?(Symbol) ? other_options.merge(value: value_field) : value_field.merge(other_options)
|
|
32
|
+
tracker_klass = RequestLogAnalyzer::Tracker.const_get(RequestLogAnalyzer.to_camelcase(tracker_klass)) if tracker_klass.is_a?(Symbol)
|
|
36
33
|
@trackers << tracker_klass.new(options)
|
|
37
34
|
end
|
|
38
35
|
end
|
|
@@ -54,7 +51,7 @@ module RequestLogAnalyzer::Aggregator
|
|
|
54
51
|
|
|
55
52
|
# Call prepare on all trackers.
|
|
56
53
|
def prepare
|
|
57
|
-
|
|
54
|
+
fail 'No trackers set up in Summarizer!' if @trackers.nil? || @trackers.empty?
|
|
58
55
|
@trackers.each { |tracker| tracker.prepare }
|
|
59
56
|
end
|
|
60
57
|
|
|
@@ -82,7 +79,7 @@ module RequestLogAnalyzer::Aggregator
|
|
|
82
79
|
# for every tracker and combines these into a single YAML export.
|
|
83
80
|
def to_yaml
|
|
84
81
|
require 'yaml'
|
|
85
|
-
trackers_export = @trackers.
|
|
82
|
+
trackers_export = @trackers.reduce({}) do |export, tracker|
|
|
86
83
|
export[tracker.title] = tracker.to_yaml_object; export
|
|
87
84
|
end
|
|
88
85
|
YAML.dump(trackers_export)
|
|
@@ -104,10 +101,10 @@ module RequestLogAnalyzer::Aggregator
|
|
|
104
101
|
# Generate report header.
|
|
105
102
|
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
|
|
106
103
|
def report_header(output)
|
|
107
|
-
output.title(
|
|
104
|
+
output.title('Request summary')
|
|
108
105
|
|
|
109
|
-
output.with_style(:
|
|
110
|
-
output.table({:
|
|
106
|
+
output.with_style(cell_separator: false) do
|
|
107
|
+
output.table({ width: 20 }, { font: :bold }) do |rows|
|
|
111
108
|
source.processed_files.each do |f|
|
|
112
109
|
rows << ['Processed File:', f]
|
|
113
110
|
end
|
|
@@ -115,7 +112,7 @@ module RequestLogAnalyzer::Aggregator
|
|
|
115
112
|
rows << ['Skipped lines:', source.skipped_lines]
|
|
116
113
|
rows << ['Parsed requests:', source.parsed_requests]
|
|
117
114
|
rows << ['Skipped requests:', source.skipped_requests]
|
|
118
|
-
rows << [
|
|
115
|
+
rows << ['Warnings:', @warnings_encountered.map { |(key, value)| "#{key}: #{value}" }.join(', ')] if has_warnings?
|
|
119
116
|
end
|
|
120
117
|
end
|
|
121
118
|
output << "\n"
|
|
@@ -125,19 +122,19 @@ module RequestLogAnalyzer::Aggregator
|
|
|
125
122
|
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
|
|
126
123
|
def report_footer(output)
|
|
127
124
|
if has_log_ordering_warnings?
|
|
128
|
-
output.title(
|
|
125
|
+
output.title('Parse warnings')
|
|
129
126
|
|
|
130
|
-
output.puts
|
|
131
|
-
output.puts
|
|
132
|
-
output.puts
|
|
133
|
-
output.puts output.link(
|
|
127
|
+
output.puts 'Parsable lines were encountered without a header line before it. It'
|
|
128
|
+
output.puts 'could be that logging is not setup correctly for your application.'
|
|
129
|
+
output.puts 'Visit this website for logging configuration tips:'
|
|
130
|
+
output.puts output.link('http://github.com/wvanbergen/request-log-analyzer/wikis/configure-logging')
|
|
134
131
|
output.puts
|
|
135
132
|
end
|
|
136
133
|
end
|
|
137
134
|
|
|
138
135
|
# Returns true if there were any warnings generated by the trackers
|
|
139
136
|
def has_warnings?
|
|
140
|
-
@warnings_encountered.
|
|
137
|
+
@warnings_encountered.reduce(0) { |result, (_, value)| result += value } > 0
|
|
141
138
|
end
|
|
142
139
|
|
|
143
140
|
# Returns true if there were any log ordering warnings
|
|
@@ -149,7 +146,7 @@ module RequestLogAnalyzer::Aggregator
|
|
|
149
146
|
# <tt>type</tt> Type of warning
|
|
150
147
|
# <tt>message</tt> Warning message
|
|
151
148
|
# <tt>lineno</tt> The line on which the error was encountered
|
|
152
|
-
def warning(type,
|
|
149
|
+
def warning(type, _message, _lineno)
|
|
153
150
|
@warnings_encountered[type] ||= 0
|
|
154
151
|
@warnings_encountered[type] += 1
|
|
155
152
|
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
module RequestLogAnalyzer
|
|
2
|
+
module ClassLevelInheritableAttributes
|
|
3
|
+
def inheritable_attributes(*args)
|
|
4
|
+
@inheritable_attributes ||= [:inheritable_attributes]
|
|
5
|
+
@inheritable_attributes += args
|
|
6
|
+
args.each do |arg|
|
|
7
|
+
class_eval %(
|
|
8
|
+
class << self; attr_accessor :#{arg} end
|
|
9
|
+
)
|
|
10
|
+
end
|
|
11
|
+
@inheritable_attributes
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def inherited(subclass)
|
|
15
|
+
@inheritable_attributes.each do |inheritable_attribute|
|
|
16
|
+
instance_var = "@#{inheritable_attribute}"
|
|
17
|
+
subclass.instance_variable_set(instance_var, instance_variable_get(instance_var))
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
super
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
module RequestLogAnalyzer
|
|
2
|
-
|
|
3
2
|
# The RequestLogAnalyzer::Controller class creates a LogParser instance for the
|
|
4
3
|
# requested file format and connect it with sources and aggregators.
|
|
5
4
|
#
|
|
@@ -14,13 +13,11 @@ module RequestLogAnalyzer
|
|
|
14
13
|
# sources are registered in the correct order. This can be helpful to parse requests
|
|
15
14
|
# from several logrotated log files.
|
|
16
15
|
class Controller
|
|
17
|
-
|
|
18
16
|
attr_reader :source, :filters, :aggregators, :output, :options
|
|
19
17
|
|
|
20
18
|
# Builds a RequestLogAnalyzer::Controller given parsed command line arguments
|
|
21
19
|
# <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
|
|
22
20
|
def self.build_from_arguments(arguments)
|
|
23
|
-
|
|
24
21
|
options = {}
|
|
25
22
|
|
|
26
23
|
# Copy fields
|
|
@@ -51,9 +48,9 @@ module RequestLogAnalyzer
|
|
|
51
48
|
|
|
52
49
|
# Apache format workaround
|
|
53
50
|
if arguments[:rails_format]
|
|
54
|
-
options[:format] = {:
|
|
51
|
+
options[:format] = { rails: arguments[:rails_format] }
|
|
55
52
|
elsif arguments[:apache_format]
|
|
56
|
-
options[:format] = {:
|
|
53
|
+
options[:format] = { apache: arguments[:apache_format] }
|
|
57
54
|
end
|
|
58
55
|
|
|
59
56
|
# Handle output format casing
|
|
@@ -149,7 +146,7 @@ module RequestLogAnalyzer
|
|
|
149
146
|
|
|
150
147
|
# Deprecation warnings
|
|
151
148
|
if options[:dump]
|
|
152
|
-
warn
|
|
149
|
+
warn '[DEPRECATION] `:dump` is deprecated. Please use `:yaml` instead.'
|
|
153
150
|
options[:yaml] = options[:dump]
|
|
154
151
|
end
|
|
155
152
|
|
|
@@ -166,21 +163,21 @@ module RequestLogAnalyzer
|
|
|
166
163
|
output_amount = options[:report_amount] == 'all' ? :all : options[:report_amount].to_i
|
|
167
164
|
|
|
168
165
|
if options[:file]
|
|
169
|
-
output_object = %w
|
|
170
|
-
output_args = {:
|
|
166
|
+
output_object = %w( File StringIO ).include?(options[:file].class.name) ? options[:file] : File.new(options[:file], 'w+')
|
|
167
|
+
output_args = { width: 80, color: false, characters: :ascii, sort: output_sort, amount: output_amount }
|
|
171
168
|
elsif options[:mail]
|
|
172
|
-
output_object = RequestLogAnalyzer::Mailer.new(options[:mail], options[:mailhost], :
|
|
173
|
-
output_args = {:
|
|
169
|
+
output_object = RequestLogAnalyzer::Mailer.new(options[:mail], options[:mailhost], subject: options[:mailsubject], from: options[:mailfrom], from_alias: options[:mailfrom_name])
|
|
170
|
+
output_args = { width: 80, color: false, characters: :ascii, sort: output_sort, amount: output_amount }
|
|
174
171
|
else
|
|
175
172
|
output_object = STDOUT
|
|
176
|
-
output_args = {:
|
|
177
|
-
:
|
|
173
|
+
output_args = { width: options[:report_width].to_i, color: !options[:boring],
|
|
174
|
+
characters: (options[:boring] ? :ascii : :utf), sort: output_sort, amount: output_amount }
|
|
178
175
|
end
|
|
179
176
|
|
|
180
177
|
output_instance = output_class.new(output_object, output_args)
|
|
181
178
|
|
|
182
179
|
# Create the controller with the correct file format
|
|
183
|
-
if options[:format].
|
|
180
|
+
if options[:format].is_a?(Hash)
|
|
184
181
|
file_format = RequestLogAnalyzer::FileFormat.load(options[:format].keys[0], options[:format].values[0])
|
|
185
182
|
else
|
|
186
183
|
file_format = RequestLogAnalyzer::FileFormat.load(options[:format])
|
|
@@ -189,15 +186,15 @@ module RequestLogAnalyzer
|
|
|
189
186
|
# Kickstart the controller
|
|
190
187
|
controller =
|
|
191
188
|
Controller.new(options[:source].new(file_format,
|
|
192
|
-
:
|
|
193
|
-
:
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
189
|
+
source_files: options[:source_files],
|
|
190
|
+
parse_strategy: options[:parse_strategy]),
|
|
191
|
+
output: output_instance,
|
|
192
|
+
database: options[:database], # FUGLY!
|
|
193
|
+
yaml: options[:yaml],
|
|
194
|
+
reset_database: options[:reset_database],
|
|
195
|
+
no_progress: options[:no_progress],
|
|
196
|
+
silent: options[:silent]
|
|
197
|
+
)
|
|
201
198
|
|
|
202
199
|
# register filters
|
|
203
200
|
if options[:after] || options[:before]
|
|
@@ -215,13 +212,13 @@ module RequestLogAnalyzer
|
|
|
215
212
|
|
|
216
213
|
if options[:reject]
|
|
217
214
|
options[:reject].each do |(field, value)|
|
|
218
|
-
controller.add_filter(:field, :
|
|
215
|
+
controller.add_filter(:field, mode: :reject, field: field, value: value)
|
|
219
216
|
end
|
|
220
217
|
end
|
|
221
218
|
|
|
222
219
|
if options[:select]
|
|
223
220
|
options[:select].each do |(field, value)|
|
|
224
|
-
controller.add_filter(:field, :
|
|
221
|
+
controller.add_filter(:field, mode: :select, field: field, value: value)
|
|
225
222
|
end
|
|
226
223
|
end
|
|
227
224
|
|
|
@@ -232,7 +229,7 @@ module RequestLogAnalyzer
|
|
|
232
229
|
controller.add_aggregator(:database_inserter) if options[:database] && !options[:aggregator].include?('database')
|
|
233
230
|
|
|
234
231
|
file_format.setup_environment(controller)
|
|
235
|
-
|
|
232
|
+
controller
|
|
236
233
|
end
|
|
237
234
|
|
|
238
235
|
# Builds a new Controller for the given log file format.
|
|
@@ -244,7 +241,6 @@ module RequestLogAnalyzer
|
|
|
244
241
|
# * <tt>:no_progress</tt> No progress bar
|
|
245
242
|
# * <tt>:silent</tt> Minimal output, only error
|
|
246
243
|
def initialize(source, options = {})
|
|
247
|
-
|
|
248
244
|
@source = source
|
|
249
245
|
@options = options
|
|
250
246
|
@aggregators = []
|
|
@@ -253,7 +249,7 @@ module RequestLogAnalyzer
|
|
|
253
249
|
@interrupted = false
|
|
254
250
|
|
|
255
251
|
# Register the request format for this session after checking its validity
|
|
256
|
-
|
|
252
|
+
fail 'Invalid file format!' unless @source.file_format.valid?
|
|
257
253
|
|
|
258
254
|
# Install event handlers for wrnings, progress updates and source changes
|
|
259
255
|
@source.warning = lambda { |type, message, lineno| @aggregators.each { |agg| agg.warning(type, message, lineno) } }
|
|
@@ -290,15 +286,15 @@ module RequestLogAnalyzer
|
|
|
290
286
|
# Adds an aggregator to the controller. The aggregator will be called for every request
|
|
291
287
|
# that is parsed from the provided sources (see add_source)
|
|
292
288
|
def add_aggregator(agg)
|
|
293
|
-
agg = RequestLogAnalyzer::Aggregator.const_get(RequestLogAnalyzer.to_camelcase(agg)) if agg.
|
|
289
|
+
agg = RequestLogAnalyzer::Aggregator.const_get(RequestLogAnalyzer.to_camelcase(agg)) if agg.is_a?(String) || agg.is_a?(Symbol)
|
|
294
290
|
@aggregators << agg.new(@source, @options)
|
|
295
291
|
end
|
|
296
292
|
|
|
297
|
-
|
|
293
|
+
alias_method :>>, :add_aggregator
|
|
298
294
|
|
|
299
295
|
# Adds a request filter to the controller.
|
|
300
296
|
def add_filter(filter, filter_options = {})
|
|
301
|
-
filter = RequestLogAnalyzer::Filter.const_get(RequestLogAnalyzer.to_camelcase(filter)) if filter.
|
|
297
|
+
filter = RequestLogAnalyzer::Filter.const_get(RequestLogAnalyzer.to_camelcase(filter)) if filter.is_a?(Symbol)
|
|
302
298
|
@filters << filter.new(source.file_format, @options.merge(filter_options))
|
|
303
299
|
end
|
|
304
300
|
|
|
@@ -310,7 +306,7 @@ module RequestLogAnalyzer
|
|
|
310
306
|
request = filter.filter(request)
|
|
311
307
|
return nil if request.nil?
|
|
312
308
|
end
|
|
313
|
-
|
|
309
|
+
request
|
|
314
310
|
end
|
|
315
311
|
|
|
316
312
|
# Push a request to all the aggregators (@aggregators).
|
|
@@ -318,7 +314,7 @@ module RequestLogAnalyzer
|
|
|
318
314
|
def aggregate_request(request)
|
|
319
315
|
return false unless request
|
|
320
316
|
@aggregators.each { |agg| agg.aggregate(request) }
|
|
321
|
-
|
|
317
|
+
true
|
|
322
318
|
end
|
|
323
319
|
|
|
324
320
|
# Runs RequestLogAnalyzer
|
|
@@ -330,7 +326,6 @@ module RequestLogAnalyzer
|
|
|
330
326
|
# 5. Call report on every aggregator
|
|
331
327
|
# 6. Finalize Source
|
|
332
328
|
def run!
|
|
333
|
-
|
|
334
329
|
# @aggregators.each{|agg| p agg}
|
|
335
330
|
|
|
336
331
|
@aggregators.each { |agg| agg.prepare }
|
|
@@ -349,27 +344,26 @@ module RequestLogAnalyzer
|
|
|
349
344
|
|
|
350
345
|
@source.finalize
|
|
351
346
|
|
|
352
|
-
if @output.io.
|
|
347
|
+
if @output.io.is_a?(File)
|
|
353
348
|
unless @options[:silent]
|
|
354
349
|
puts
|
|
355
|
-
puts
|
|
356
|
-
puts
|
|
357
|
-
puts
|
|
358
|
-
puts
|
|
350
|
+
puts 'Report written to: ' + File.expand_path(@output.io.path)
|
|
351
|
+
puts 'Need an expert to analyze your application?'
|
|
352
|
+
puts 'Mail to contact@railsdoctors.com or visit us at http://railsdoctors.com'
|
|
353
|
+
puts 'Thanks for using request-log-analyzer!'
|
|
359
354
|
end
|
|
360
355
|
@output.io.close
|
|
361
|
-
elsif @output.io.
|
|
356
|
+
elsif @output.io.is_a?(RequestLogAnalyzer::Mailer)
|
|
362
357
|
@output.io.mail
|
|
363
358
|
end
|
|
364
359
|
end
|
|
365
360
|
|
|
366
361
|
def install_signal_handlers
|
|
367
|
-
Signal.trap(
|
|
362
|
+
Signal.trap('INT') do
|
|
368
363
|
handle_progress(:interrupted)
|
|
369
|
-
puts
|
|
364
|
+
puts 'Caught interrupt! Stopping parsing...'
|
|
370
365
|
@interrupted = true
|
|
371
366
|
end
|
|
372
367
|
end
|
|
373
|
-
|
|
374
368
|
end
|
|
375
369
|
end
|