request-log-analyzer 1.13.1 → 1.13.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/bin/console +17 -0
- data/lib/cli/command_line_arguments.rb +29 -36
- data/lib/cli/database_console.rb +1 -3
- data/lib/cli/database_console_init.rb +11 -11
- data/lib/cli/progressbar.rb +30 -32
- data/lib/cli/tools.rb +20 -23
- data/lib/request_log_analyzer.rb +8 -8
- data/lib/request_log_analyzer/aggregator.rb +4 -7
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +10 -13
- data/lib/request_log_analyzer/aggregator/echo.rb +5 -7
- data/lib/request_log_analyzer/aggregator/summarizer.rb +15 -18
- data/lib/request_log_analyzer/class_level_inheritable_attributes.rb +23 -0
- data/lib/request_log_analyzer/controller.rb +36 -42
- data/lib/request_log_analyzer/database.rb +4 -6
- data/lib/request_log_analyzer/database/base.rb +39 -41
- data/lib/request_log_analyzer/database/connection.rb +8 -10
- data/lib/request_log_analyzer/database/request.rb +1 -3
- data/lib/request_log_analyzer/database/source.rb +0 -2
- data/lib/request_log_analyzer/database/warning.rb +4 -6
- data/lib/request_log_analyzer/file_format.rb +46 -49
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +15 -19
- data/lib/request_log_analyzer/file_format/apache.rb +42 -45
- data/lib/request_log_analyzer/file_format/delayed_job.rb +13 -15
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job3.rb +5 -8
- data/lib/request_log_analyzer/file_format/delayed_job4.rb +5 -8
- data/lib/request_log_analyzer/file_format/haproxy.rb +44 -48
- data/lib/request_log_analyzer/file_format/merb.rb +13 -17
- data/lib/request_log_analyzer/file_format/mysql.rb +21 -25
- data/lib/request_log_analyzer/file_format/nginx.rb +0 -2
- data/lib/request_log_analyzer/file_format/oink.rb +30 -31
- data/lib/request_log_analyzer/file_format/postgresql.rb +11 -15
- data/lib/request_log_analyzer/file_format/rack.rb +0 -2
- data/lib/request_log_analyzer/file_format/rails.rb +100 -104
- data/lib/request_log_analyzer/file_format/rails3.rb +19 -23
- data/lib/request_log_analyzer/file_format/rails_development.rb +0 -1
- data/lib/request_log_analyzer/file_format/w3c.rb +16 -18
- data/lib/request_log_analyzer/filter.rb +0 -2
- data/lib/request_log_analyzer/filter/anonymize.rb +4 -7
- data/lib/request_log_analyzer/filter/field.rb +3 -6
- data/lib/request_log_analyzer/filter/timespan.rb +2 -6
- data/lib/request_log_analyzer/line_definition.rb +16 -19
- data/lib/request_log_analyzer/log_processor.rb +10 -14
- data/lib/request_log_analyzer/mailer.rb +9 -12
- data/lib/request_log_analyzer/output.rb +12 -14
- data/lib/request_log_analyzer/output/fixed_width.rb +21 -28
- data/lib/request_log_analyzer/output/html.rb +11 -14
- data/lib/request_log_analyzer/request.rb +53 -33
- data/lib/request_log_analyzer/source.rb +2 -5
- data/lib/request_log_analyzer/source/log_parser.rb +9 -16
- data/lib/request_log_analyzer/tracker.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +4 -6
- data/lib/request_log_analyzer/tracker/frequency.rb +9 -11
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -11
- data/lib/request_log_analyzer/tracker/numeric_value.rb +40 -44
- data/lib/request_log_analyzer/tracker/timespan.rb +5 -8
- data/lib/request_log_analyzer/tracker/traffic.rb +8 -10
- data/lib/request_log_analyzer/version.rb +1 -1
- data/request-log-analyzer.gemspec +6 -6
- data/spec/integration/command_line_usage_spec.rb +33 -33
- data/spec/integration/mailer_spec.rb +181 -185
- data/spec/integration/munin_plugins_rails_spec.rb +20 -20
- data/spec/integration/scout_spec.rb +40 -41
- data/spec/lib/helpers.rb +8 -9
- data/spec/lib/macros.rb +2 -4
- data/spec/lib/matchers.rb +20 -25
- data/spec/lib/mocks.rb +10 -11
- data/spec/lib/testing_format.rb +8 -10
- data/spec/spec_helper.rb +5 -1
- data/spec/unit/aggregator/database_inserter_spec.rb +23 -23
- data/spec/unit/aggregator/summarizer_spec.rb +7 -7
- data/spec/unit/controller/controller_spec.rb +14 -14
- data/spec/unit/controller/log_processor_spec.rb +3 -3
- data/spec/unit/database/base_class_spec.rb +36 -37
- data/spec/unit/database/connection_spec.rb +10 -10
- data/spec/unit/database/database_spec.rb +11 -11
- data/spec/unit/file_format/amazon_s3_format_spec.rb +66 -62
- data/spec/unit/file_format/apache_format_spec.rb +57 -52
- data/spec/unit/file_format/common_regular_expressions_spec.rb +18 -21
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job2_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job3_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job4_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job_format_spec.rb +12 -12
- data/spec/unit/file_format/file_format_api_spec.rb +19 -19
- data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
- data/spec/unit/file_format/haproxy_format_spec.rb +53 -49
- data/spec/unit/file_format/inheritance_spec.rb +13 -0
- data/spec/unit/file_format/line_definition_spec.rb +35 -33
- data/spec/unit/file_format/merb_format_spec.rb +13 -11
- data/spec/unit/file_format/mysql_format_spec.rb +24 -24
- data/spec/unit/file_format/oink_format_spec.rb +29 -29
- data/spec/unit/file_format/postgresql_format_spec.rb +9 -9
- data/spec/unit/file_format/rack_format_spec.rb +36 -31
- data/spec/unit/file_format/rails3_format_spec.rb +46 -46
- data/spec/unit/file_format/rails_format_spec.rb +52 -53
- data/spec/unit/file_format/w3c_format_spec.rb +27 -24
- data/spec/unit/filter/anonymize_filter_spec.rb +7 -7
- data/spec/unit/filter/field_filter_spec.rb +26 -26
- data/spec/unit/filter/filter_spec.rb +4 -4
- data/spec/unit/filter/timespan_filter_spec.rb +22 -22
- data/spec/unit/mailer_spec.rb +21 -21
- data/spec/unit/request_spec.rb +29 -29
- data/spec/unit/source/log_parser_spec.rb +5 -5
- data/spec/unit/tracker/duration_tracker_spec.rb +23 -23
- data/spec/unit/tracker/frequency_tracker_spec.rb +29 -30
- data/spec/unit/tracker/hourly_spread_spec.rb +35 -35
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +71 -72
- data/spec/unit/tracker/timespan_tracker_spec.rb +31 -31
- data/spec/unit/tracker/tracker_api_spec.rb +43 -44
- data/spec/unit/tracker/traffic_tracker_spec.rb +7 -7
- metadata +38 -35
data/lib/cli/tools.rb
CHANGED
@@ -6,27 +6,24 @@ module CommandLine
|
|
6
6
|
# If it is not possible to to so, it returns the default_width.
|
7
7
|
# <tt>default_width</tt> Defaults to 81
|
8
8
|
def terminal_width(default_width = 81, out = STDOUT)
|
9
|
-
|
9
|
+
tiocgwinsz = 0x5413
|
10
|
+
data = [0, 0, 0, 0].pack('SSSS')
|
11
|
+
if !RUBY_PLATFORM.include?('java') && out.ioctl(tiocgwinsz, data) >= 0 # JRuby crashes on ioctl
|
12
|
+
_, cols, _, _ = data.unpack('SSSS')
|
13
|
+
fail unless cols > 0
|
14
|
+
cols
|
15
|
+
else
|
16
|
+
fail
|
17
|
+
end
|
18
|
+
rescue
|
10
19
|
begin
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
raise unless cols > 0
|
16
|
-
cols
|
17
|
-
else
|
18
|
-
raise
|
20
|
+
IO.popen('stty -a 2>&1') do |pipe|
|
21
|
+
column_line = pipe.find { |line| /(\d+) columns/ =~ line }
|
22
|
+
fail unless column_line
|
23
|
+
Regexp.last_match[1].to_i
|
19
24
|
end
|
20
25
|
rescue
|
21
|
-
|
22
|
-
IO.popen('stty -a 2>&1') do |pipe|
|
23
|
-
column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
|
24
|
-
raise unless column_line
|
25
|
-
$1.to_i
|
26
|
-
end
|
27
|
-
rescue
|
28
|
-
default_width
|
29
|
-
end
|
26
|
+
default_width
|
30
27
|
end
|
31
28
|
end
|
32
29
|
|
@@ -40,14 +37,14 @@ module CommandLine
|
|
40
37
|
if File.directory?('./lib/tasks/')
|
41
38
|
task_file = File.expand_path('../../tasks/request_log_analyzer.rake', File.dirname(__FILE__))
|
42
39
|
FileUtils.copy(task_file, './lib/tasks/request_log_analyze.rake')
|
43
|
-
puts
|
44
|
-
puts
|
40
|
+
puts 'Installed rake tasks.'
|
41
|
+
puts 'To use, run: rake rla:report'
|
45
42
|
else
|
46
|
-
puts
|
47
|
-
puts
|
43
|
+
puts 'Cannot find /lib/tasks folder. Are you in your Rails directory?'
|
44
|
+
puts 'Installation aborted.'
|
48
45
|
end
|
49
46
|
else
|
50
|
-
|
47
|
+
fail "Cannot perform this install type! (#{install_type})"
|
51
48
|
end
|
52
49
|
end
|
53
50
|
end
|
data/lib/request_log_analyzer.rb
CHANGED
@@ -1,37 +1,37 @@
|
|
1
1
|
require 'date'
|
2
2
|
|
3
3
|
# RequestLogAnalyzer is the base namespace in which all functionality of RequestLogAnalyzer is implemented.
|
4
|
-
# This module
|
4
|
+
# This module itself contains some functions to help with class and source file loading. The actual
|
5
5
|
# application startup code resides in the {RequestLogAnalyzer::Controller} class.
|
6
6
|
#
|
7
7
|
# The {RequestLogAnalyzer::VERSION} constant can be used to determine what version of request-log-analyzer
|
8
8
|
# is running.
|
9
9
|
module RequestLogAnalyzer
|
10
|
-
|
11
|
-
#
|
12
|
-
# (<tt>request_log_analyzer/controller</tt>). This function can be used to load the file (using
|
10
|
+
# Convert a string/symbol in camel case ({RequestLogAnalyzer::Controller}) to underscores
|
11
|
+
# (<tt>request_log_analyzer/controller</tt>). This function can be used to load the file (using
|
13
12
|
# <tt>require</tt>) in which the given constant is defined.
|
14
13
|
#
|
15
14
|
# @param [#to_s] str The string-like to convert in the following format: <tt>ModuleName::ClassName</tt>.
|
16
15
|
# @return [String] The input string converted to underscore form.
|
17
16
|
def self.to_underscore(str)
|
18
|
-
str.to_s.gsub(/::/, '/').gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').gsub(/([a-z\d])([A-Z])/,'\1_\2').tr(
|
17
|
+
str.to_s.gsub(/::/, '/').gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').gsub(/([a-z\d])([A-Z])/, '\1_\2').tr('-', '_').downcase
|
19
18
|
end
|
20
19
|
|
21
20
|
# Convert a string/symbol in underscores (<tt>request_log_analyzer/controller</tt>) to camelcase
|
22
|
-
# ({RequestLogAnalyzer::Controller}). This can be used to find the class that is defined in a given
|
21
|
+
# ({RequestLogAnalyzer::Controller}). This can be used to find the class that is defined in a given
|
23
22
|
# filename.
|
24
23
|
#
|
25
24
|
# @param [#to_s] str The string-like to convert in the f`ollowing format: <tt>module_name/class_name</tt>.
|
26
|
-
# @return [String] The input string converted to camelcase form.
|
25
|
+
# @return [String] The input string converted to camelcase form.
|
27
26
|
def self.to_camelcase(str)
|
28
|
-
str.to_s.gsub(/\/(.?)/) {
|
27
|
+
str.to_s.gsub(/\/(.?)/) { '::' + Regexp.last_match[1].upcase }.gsub(/(^|_)(.)/) { Regexp.last_match[2].upcase }
|
29
28
|
end
|
30
29
|
end
|
31
30
|
|
32
31
|
require 'request_log_analyzer/version'
|
33
32
|
require 'request_log_analyzer/controller'
|
34
33
|
require 'request_log_analyzer/aggregator'
|
34
|
+
require 'request_log_analyzer/class_level_inheritable_attributes'
|
35
35
|
require 'request_log_analyzer/file_format'
|
36
36
|
require 'request_log_analyzer/filter'
|
37
37
|
require 'request_log_analyzer/line_definition'
|
@@ -1,9 +1,7 @@
|
|
1
1
|
module RequestLogAnalyzer::Aggregator
|
2
|
-
|
3
2
|
# The base class of an aggregator. This class provides the interface to which
|
4
3
|
# every aggregator should comply (by simply subclassing this class).
|
5
4
|
class Base
|
6
|
-
|
7
5
|
attr_reader :options, :source
|
8
6
|
|
9
7
|
# Intializes a new RequestLogAnalyzer::Aggregator::Base instance
|
@@ -20,7 +18,7 @@ module RequestLogAnalyzer::Aggregator
|
|
20
18
|
|
21
19
|
# The aggregate function is called for every request.
|
22
20
|
# Implement the aggregating functionality in this method
|
23
|
-
def aggregate(
|
21
|
+
def aggregate(_request)
|
24
22
|
end
|
25
23
|
|
26
24
|
# The finalize function is called after all sources are parsed and no more
|
@@ -29,18 +27,17 @@ module RequestLogAnalyzer::Aggregator
|
|
29
27
|
end
|
30
28
|
|
31
29
|
# The warning method is called if the parser eits a warning.
|
32
|
-
def warning(
|
30
|
+
def warning(_type, _message, _lineno)
|
33
31
|
end
|
34
32
|
|
35
33
|
# The report function is called at the end. Implement any result reporting
|
36
34
|
# in this function.
|
37
|
-
def report(
|
35
|
+
def report(_output)
|
38
36
|
end
|
39
37
|
|
40
38
|
# The source_change function gets called when handling a source is started or finished.
|
41
|
-
def source_change(
|
39
|
+
def source_change(_change, _filename)
|
42
40
|
end
|
43
|
-
|
44
41
|
end
|
45
42
|
end
|
46
43
|
|
@@ -1,5 +1,4 @@
|
|
1
1
|
module RequestLogAnalyzer::Aggregator
|
2
|
-
|
3
2
|
# The database aggregator will create an SQLite3 database with all parsed request information.
|
4
3
|
#
|
5
4
|
# The prepare method will create a database schema according to the file format definitions.
|
@@ -12,14 +11,13 @@ module RequestLogAnalyzer::Aggregator
|
|
12
11
|
# the request record, and a field for every parsed value. Finally, a warnings table will be
|
13
12
|
# created to log all parse warnings.
|
14
13
|
class DatabaseInserter < Base
|
15
|
-
|
16
14
|
attr_reader :request_count, :sources, :database
|
17
15
|
|
18
16
|
# Establishes a connection to the database and creates the necessary database schema for the
|
19
17
|
# current file format
|
20
18
|
def prepare
|
21
19
|
require 'request_log_analyzer/database'
|
22
|
-
|
20
|
+
|
23
21
|
@sources = {}
|
24
22
|
@database = RequestLogAnalyzer::Database.new(options[:database])
|
25
23
|
@database.file_format = source.file_format
|
@@ -32,14 +30,14 @@ module RequestLogAnalyzer::Aggregator
|
|
32
30
|
# This will create a record in the requests table and create a record for every line that has been parsed,
|
33
31
|
# in which the captured values will be stored.
|
34
32
|
def aggregate(request)
|
35
|
-
@request_object = RequestLogAnalyzer::Database::Request.new(:
|
33
|
+
@request_object = RequestLogAnalyzer::Database::Request.new(first_lineno: request.first_lineno, last_lineno: request.last_lineno)
|
36
34
|
request.lines.each do |line|
|
37
|
-
class_columns = database.get_class(line[:line_type]).column_names.reject { |column|
|
38
|
-
attributes = Hash[*line.select { |(key, _)| class_columns.include?(key.to_s)}.flatten]
|
39
|
-
|
35
|
+
class_columns = database.get_class(line[:line_type]).column_names.reject { |column| %w(id source_id request_id).include?(column) }
|
36
|
+
attributes = Hash[*line.select { |(key, _)| class_columns.include?(key.to_s) }.flatten]
|
37
|
+
|
40
38
|
# Fix encoding patch for 1.9.2
|
41
|
-
attributes.each do |k,v|
|
42
|
-
attributes[k] = v.force_encoding(
|
39
|
+
attributes.each do |k, v|
|
40
|
+
attributes[k] = v.force_encoding('UTF-8') if v.is_a?(String)
|
43
41
|
end
|
44
42
|
|
45
43
|
@request_object.send("#{line[:line_type]}_lines").build(attributes)
|
@@ -58,7 +56,7 @@ module RequestLogAnalyzer::Aggregator
|
|
58
56
|
|
59
57
|
# Records w warining in the warnings table.
|
60
58
|
def warning(type, message, lineno)
|
61
|
-
RequestLogAnalyzer::Database::Warning.create!(:
|
59
|
+
RequestLogAnalyzer::Database::Warning.create!(warning_type: type.to_s, message: message, lineno: lineno)
|
62
60
|
end
|
63
61
|
|
64
62
|
# Records source changes in the sources table
|
@@ -66,9 +64,9 @@ module RequestLogAnalyzer::Aggregator
|
|
66
64
|
if File.exist?(filename)
|
67
65
|
case change
|
68
66
|
when :started
|
69
|
-
@sources[filename] = RequestLogAnalyzer::Database::Source.create!(:
|
67
|
+
@sources[filename] = RequestLogAnalyzer::Database::Source.create!(filename: filename)
|
70
68
|
when :finished
|
71
|
-
@sources[filename].update_attributes!(:
|
69
|
+
@sources[filename].update_attributes!(filesize: File.size(filename), mtime: File.mtime(filename))
|
72
70
|
end
|
73
71
|
end
|
74
72
|
end
|
@@ -84,6 +82,5 @@ module RequestLogAnalyzer::Aggregator
|
|
84
82
|
output << output.colorize(" $ request-log-analyzer console -d #{options[:database]}\n", :bold)
|
85
83
|
output << "\n"
|
86
84
|
end
|
87
|
-
|
88
85
|
end
|
89
86
|
end
|
@@ -1,8 +1,6 @@
|
|
1
1
|
module RequestLogAnalyzer::Aggregator
|
2
|
-
|
3
2
|
# Echo Aggregator. Writes everything to the screen when it is passed to this aggregator
|
4
3
|
class Echo < Base
|
5
|
-
|
6
4
|
attr_accessor :warnings
|
7
5
|
|
8
6
|
def prepare
|
@@ -11,8 +9,9 @@ module RequestLogAnalyzer::Aggregator
|
|
11
9
|
|
12
10
|
# Display every parsed line immediately to the terminal
|
13
11
|
def aggregate(request)
|
14
|
-
puts "\nRequest: \n" + request.lines.map
|
15
|
-
"\t#{l[:lineno]}:#{l[:line_type]}: #{l.reject { |(k,_)| [:lineno, :line_type].include?(k) }.inspect}"
|
12
|
+
puts "\nRequest: \n" + request.lines.map do |l|
|
13
|
+
"\t#{l[:lineno]}:#{l[:line_type]}: #{l.reject { |(k, _)| [:lineno, :line_type].include?(k) }.inspect}"
|
14
|
+
end.join("\n")
|
16
15
|
end
|
17
16
|
|
18
17
|
# Capture all warnings during parsing
|
@@ -22,9 +21,8 @@ module RequestLogAnalyzer::Aggregator
|
|
22
21
|
|
23
22
|
# Display every warning in the report when finished parsing
|
24
23
|
def report(output)
|
25
|
-
output.title(
|
24
|
+
output.title('Warnings during parsing')
|
26
25
|
@warnings.each { |w| output.puts(w) }
|
27
26
|
end
|
28
|
-
|
29
27
|
end
|
30
|
-
end
|
28
|
+
end
|
@@ -1,9 +1,6 @@
|
|
1
1
|
module RequestLogAnalyzer::Aggregator
|
2
|
-
|
3
2
|
class Summarizer < Base
|
4
|
-
|
5
3
|
class Definer
|
6
|
-
|
7
4
|
attr_reader :trackers
|
8
5
|
|
9
6
|
# Initialize tracker array
|
@@ -31,8 +28,8 @@ module RequestLogAnalyzer::Aggregator
|
|
31
28
|
# <tt>tracker_class</tt> The class to include
|
32
29
|
# <tt>optiont</tt> The options to pass to the trackers.
|
33
30
|
def track(tracker_klass, value_field = {}, other_options = {})
|
34
|
-
options = value_field.
|
35
|
-
tracker_klass = RequestLogAnalyzer::Tracker.const_get(RequestLogAnalyzer.to_camelcase(tracker_klass)) if tracker_klass.
|
31
|
+
options = value_field.is_a?(Symbol) ? other_options.merge(value: value_field) : value_field.merge(other_options)
|
32
|
+
tracker_klass = RequestLogAnalyzer::Tracker.const_get(RequestLogAnalyzer.to_camelcase(tracker_klass)) if tracker_klass.is_a?(Symbol)
|
36
33
|
@trackers << tracker_klass.new(options)
|
37
34
|
end
|
38
35
|
end
|
@@ -54,7 +51,7 @@ module RequestLogAnalyzer::Aggregator
|
|
54
51
|
|
55
52
|
# Call prepare on all trackers.
|
56
53
|
def prepare
|
57
|
-
|
54
|
+
fail 'No trackers set up in Summarizer!' if @trackers.nil? || @trackers.empty?
|
58
55
|
@trackers.each { |tracker| tracker.prepare }
|
59
56
|
end
|
60
57
|
|
@@ -82,7 +79,7 @@ module RequestLogAnalyzer::Aggregator
|
|
82
79
|
# for every tracker and combines these into a single YAML export.
|
83
80
|
def to_yaml
|
84
81
|
require 'yaml'
|
85
|
-
trackers_export = @trackers.
|
82
|
+
trackers_export = @trackers.reduce({}) do |export, tracker|
|
86
83
|
export[tracker.title] = tracker.to_yaml_object; export
|
87
84
|
end
|
88
85
|
YAML.dump(trackers_export)
|
@@ -104,10 +101,10 @@ module RequestLogAnalyzer::Aggregator
|
|
104
101
|
# Generate report header.
|
105
102
|
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
|
106
103
|
def report_header(output)
|
107
|
-
output.title(
|
104
|
+
output.title('Request summary')
|
108
105
|
|
109
|
-
output.with_style(:
|
110
|
-
output.table({:
|
106
|
+
output.with_style(cell_separator: false) do
|
107
|
+
output.table({ width: 20 }, { font: :bold }) do |rows|
|
111
108
|
source.processed_files.each do |f|
|
112
109
|
rows << ['Processed File:', f]
|
113
110
|
end
|
@@ -115,7 +112,7 @@ module RequestLogAnalyzer::Aggregator
|
|
115
112
|
rows << ['Skipped lines:', source.skipped_lines]
|
116
113
|
rows << ['Parsed requests:', source.parsed_requests]
|
117
114
|
rows << ['Skipped requests:', source.skipped_requests]
|
118
|
-
rows << [
|
115
|
+
rows << ['Warnings:', @warnings_encountered.map { |(key, value)| "#{key}: #{value}" }.join(', ')] if has_warnings?
|
119
116
|
end
|
120
117
|
end
|
121
118
|
output << "\n"
|
@@ -125,19 +122,19 @@ module RequestLogAnalyzer::Aggregator
|
|
125
122
|
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
|
126
123
|
def report_footer(output)
|
127
124
|
if has_log_ordering_warnings?
|
128
|
-
output.title(
|
125
|
+
output.title('Parse warnings')
|
129
126
|
|
130
|
-
output.puts
|
131
|
-
output.puts
|
132
|
-
output.puts
|
133
|
-
output.puts output.link(
|
127
|
+
output.puts 'Parsable lines were encountered without a header line before it. It'
|
128
|
+
output.puts 'could be that logging is not setup correctly for your application.'
|
129
|
+
output.puts 'Visit this website for logging configuration tips:'
|
130
|
+
output.puts output.link('http://github.com/wvanbergen/request-log-analyzer/wikis/configure-logging')
|
134
131
|
output.puts
|
135
132
|
end
|
136
133
|
end
|
137
134
|
|
138
135
|
# Returns true if there were any warnings generated by the trackers
|
139
136
|
def has_warnings?
|
140
|
-
@warnings_encountered.
|
137
|
+
@warnings_encountered.reduce(0) { |result, (_, value)| result += value } > 0
|
141
138
|
end
|
142
139
|
|
143
140
|
# Returns true if there were any log ordering warnings
|
@@ -149,7 +146,7 @@ module RequestLogAnalyzer::Aggregator
|
|
149
146
|
# <tt>type</tt> Type of warning
|
150
147
|
# <tt>message</tt> Warning message
|
151
148
|
# <tt>lineno</tt> The line on which the error was encountered
|
152
|
-
def warning(type,
|
149
|
+
def warning(type, _message, _lineno)
|
153
150
|
@warnings_encountered[type] ||= 0
|
154
151
|
@warnings_encountered[type] += 1
|
155
152
|
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module RequestLogAnalyzer
|
2
|
+
module ClassLevelInheritableAttributes
|
3
|
+
def inheritable_attributes(*args)
|
4
|
+
@inheritable_attributes ||= [:inheritable_attributes]
|
5
|
+
@inheritable_attributes += args
|
6
|
+
args.each do |arg|
|
7
|
+
class_eval %(
|
8
|
+
class << self; attr_accessor :#{arg} end
|
9
|
+
)
|
10
|
+
end
|
11
|
+
@inheritable_attributes
|
12
|
+
end
|
13
|
+
|
14
|
+
def inherited(subclass)
|
15
|
+
@inheritable_attributes.each do |inheritable_attribute|
|
16
|
+
instance_var = "@#{inheritable_attribute}"
|
17
|
+
subclass.instance_variable_set(instance_var, instance_variable_get(instance_var))
|
18
|
+
end
|
19
|
+
|
20
|
+
super
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -1,5 +1,4 @@
|
|
1
1
|
module RequestLogAnalyzer
|
2
|
-
|
3
2
|
# The RequestLogAnalyzer::Controller class creates a LogParser instance for the
|
4
3
|
# requested file format and connect it with sources and aggregators.
|
5
4
|
#
|
@@ -14,13 +13,11 @@ module RequestLogAnalyzer
|
|
14
13
|
# sources are registered in the correct order. This can be helpful to parse requests
|
15
14
|
# from several logrotated log files.
|
16
15
|
class Controller
|
17
|
-
|
18
16
|
attr_reader :source, :filters, :aggregators, :output, :options
|
19
17
|
|
20
18
|
# Builds a RequestLogAnalyzer::Controller given parsed command line arguments
|
21
19
|
# <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
|
22
20
|
def self.build_from_arguments(arguments)
|
23
|
-
|
24
21
|
options = {}
|
25
22
|
|
26
23
|
# Copy fields
|
@@ -51,9 +48,9 @@ module RequestLogAnalyzer
|
|
51
48
|
|
52
49
|
# Apache format workaround
|
53
50
|
if arguments[:rails_format]
|
54
|
-
options[:format] = {:
|
51
|
+
options[:format] = { rails: arguments[:rails_format] }
|
55
52
|
elsif arguments[:apache_format]
|
56
|
-
options[:format] = {:
|
53
|
+
options[:format] = { apache: arguments[:apache_format] }
|
57
54
|
end
|
58
55
|
|
59
56
|
# Handle output format casing
|
@@ -149,7 +146,7 @@ module RequestLogAnalyzer
|
|
149
146
|
|
150
147
|
# Deprecation warnings
|
151
148
|
if options[:dump]
|
152
|
-
warn
|
149
|
+
warn '[DEPRECATION] `:dump` is deprecated. Please use `:yaml` instead.'
|
153
150
|
options[:yaml] = options[:dump]
|
154
151
|
end
|
155
152
|
|
@@ -166,21 +163,21 @@ module RequestLogAnalyzer
|
|
166
163
|
output_amount = options[:report_amount] == 'all' ? :all : options[:report_amount].to_i
|
167
164
|
|
168
165
|
if options[:file]
|
169
|
-
output_object = %w
|
170
|
-
output_args = {:
|
166
|
+
output_object = %w( File StringIO ).include?(options[:file].class.name) ? options[:file] : File.new(options[:file], 'w+')
|
167
|
+
output_args = { width: 80, color: false, characters: :ascii, sort: output_sort, amount: output_amount }
|
171
168
|
elsif options[:mail]
|
172
|
-
output_object = RequestLogAnalyzer::Mailer.new(options[:mail], options[:mailhost], :
|
173
|
-
output_args = {:
|
169
|
+
output_object = RequestLogAnalyzer::Mailer.new(options[:mail], options[:mailhost], subject: options[:mailsubject], from: options[:mailfrom], from_alias: options[:mailfrom_name])
|
170
|
+
output_args = { width: 80, color: false, characters: :ascii, sort: output_sort, amount: output_amount }
|
174
171
|
else
|
175
172
|
output_object = STDOUT
|
176
|
-
output_args = {:
|
177
|
-
:
|
173
|
+
output_args = { width: options[:report_width].to_i, color: !options[:boring],
|
174
|
+
characters: (options[:boring] ? :ascii : :utf), sort: output_sort, amount: output_amount }
|
178
175
|
end
|
179
176
|
|
180
177
|
output_instance = output_class.new(output_object, output_args)
|
181
178
|
|
182
179
|
# Create the controller with the correct file format
|
183
|
-
if options[:format].
|
180
|
+
if options[:format].is_a?(Hash)
|
184
181
|
file_format = RequestLogAnalyzer::FileFormat.load(options[:format].keys[0], options[:format].values[0])
|
185
182
|
else
|
186
183
|
file_format = RequestLogAnalyzer::FileFormat.load(options[:format])
|
@@ -189,15 +186,15 @@ module RequestLogAnalyzer
|
|
189
186
|
# Kickstart the controller
|
190
187
|
controller =
|
191
188
|
Controller.new(options[:source].new(file_format,
|
192
|
-
:
|
193
|
-
:
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
189
|
+
source_files: options[:source_files],
|
190
|
+
parse_strategy: options[:parse_strategy]),
|
191
|
+
output: output_instance,
|
192
|
+
database: options[:database], # FUGLY!
|
193
|
+
yaml: options[:yaml],
|
194
|
+
reset_database: options[:reset_database],
|
195
|
+
no_progress: options[:no_progress],
|
196
|
+
silent: options[:silent]
|
197
|
+
)
|
201
198
|
|
202
199
|
# register filters
|
203
200
|
if options[:after] || options[:before]
|
@@ -215,13 +212,13 @@ module RequestLogAnalyzer
|
|
215
212
|
|
216
213
|
if options[:reject]
|
217
214
|
options[:reject].each do |(field, value)|
|
218
|
-
controller.add_filter(:field, :
|
215
|
+
controller.add_filter(:field, mode: :reject, field: field, value: value)
|
219
216
|
end
|
220
217
|
end
|
221
218
|
|
222
219
|
if options[:select]
|
223
220
|
options[:select].each do |(field, value)|
|
224
|
-
controller.add_filter(:field, :
|
221
|
+
controller.add_filter(:field, mode: :select, field: field, value: value)
|
225
222
|
end
|
226
223
|
end
|
227
224
|
|
@@ -232,7 +229,7 @@ module RequestLogAnalyzer
|
|
232
229
|
controller.add_aggregator(:database_inserter) if options[:database] && !options[:aggregator].include?('database')
|
233
230
|
|
234
231
|
file_format.setup_environment(controller)
|
235
|
-
|
232
|
+
controller
|
236
233
|
end
|
237
234
|
|
238
235
|
# Builds a new Controller for the given log file format.
|
@@ -244,7 +241,6 @@ module RequestLogAnalyzer
|
|
244
241
|
# * <tt>:no_progress</tt> No progress bar
|
245
242
|
# * <tt>:silent</tt> Minimal output, only error
|
246
243
|
def initialize(source, options = {})
|
247
|
-
|
248
244
|
@source = source
|
249
245
|
@options = options
|
250
246
|
@aggregators = []
|
@@ -253,7 +249,7 @@ module RequestLogAnalyzer
|
|
253
249
|
@interrupted = false
|
254
250
|
|
255
251
|
# Register the request format for this session after checking its validity
|
256
|
-
|
252
|
+
fail 'Invalid file format!' unless @source.file_format.valid?
|
257
253
|
|
258
254
|
# Install event handlers for wrnings, progress updates and source changes
|
259
255
|
@source.warning = lambda { |type, message, lineno| @aggregators.each { |agg| agg.warning(type, message, lineno) } }
|
@@ -290,15 +286,15 @@ module RequestLogAnalyzer
|
|
290
286
|
# Adds an aggregator to the controller. The aggregator will be called for every request
|
291
287
|
# that is parsed from the provided sources (see add_source)
|
292
288
|
def add_aggregator(agg)
|
293
|
-
agg = RequestLogAnalyzer::Aggregator.const_get(RequestLogAnalyzer.to_camelcase(agg)) if agg.
|
289
|
+
agg = RequestLogAnalyzer::Aggregator.const_get(RequestLogAnalyzer.to_camelcase(agg)) if agg.is_a?(String) || agg.is_a?(Symbol)
|
294
290
|
@aggregators << agg.new(@source, @options)
|
295
291
|
end
|
296
292
|
|
297
|
-
|
293
|
+
alias_method :>>, :add_aggregator
|
298
294
|
|
299
295
|
# Adds a request filter to the controller.
|
300
296
|
def add_filter(filter, filter_options = {})
|
301
|
-
filter = RequestLogAnalyzer::Filter.const_get(RequestLogAnalyzer.to_camelcase(filter)) if filter.
|
297
|
+
filter = RequestLogAnalyzer::Filter.const_get(RequestLogAnalyzer.to_camelcase(filter)) if filter.is_a?(Symbol)
|
302
298
|
@filters << filter.new(source.file_format, @options.merge(filter_options))
|
303
299
|
end
|
304
300
|
|
@@ -310,7 +306,7 @@ module RequestLogAnalyzer
|
|
310
306
|
request = filter.filter(request)
|
311
307
|
return nil if request.nil?
|
312
308
|
end
|
313
|
-
|
309
|
+
request
|
314
310
|
end
|
315
311
|
|
316
312
|
# Push a request to all the aggregators (@aggregators).
|
@@ -318,7 +314,7 @@ module RequestLogAnalyzer
|
|
318
314
|
def aggregate_request(request)
|
319
315
|
return false unless request
|
320
316
|
@aggregators.each { |agg| agg.aggregate(request) }
|
321
|
-
|
317
|
+
true
|
322
318
|
end
|
323
319
|
|
324
320
|
# Runs RequestLogAnalyzer
|
@@ -330,7 +326,6 @@ module RequestLogAnalyzer
|
|
330
326
|
# 5. Call report on every aggregator
|
331
327
|
# 6. Finalize Source
|
332
328
|
def run!
|
333
|
-
|
334
329
|
# @aggregators.each{|agg| p agg}
|
335
330
|
|
336
331
|
@aggregators.each { |agg| agg.prepare }
|
@@ -349,27 +344,26 @@ module RequestLogAnalyzer
|
|
349
344
|
|
350
345
|
@source.finalize
|
351
346
|
|
352
|
-
if @output.io.
|
347
|
+
if @output.io.is_a?(File)
|
353
348
|
unless @options[:silent]
|
354
349
|
puts
|
355
|
-
puts
|
356
|
-
puts
|
357
|
-
puts
|
358
|
-
puts
|
350
|
+
puts 'Report written to: ' + File.expand_path(@output.io.path)
|
351
|
+
puts 'Need an expert to analyze your application?'
|
352
|
+
puts 'Mail to contact@railsdoctors.com or visit us at http://railsdoctors.com'
|
353
|
+
puts 'Thanks for using request-log-analyzer!'
|
359
354
|
end
|
360
355
|
@output.io.close
|
361
|
-
elsif @output.io.
|
356
|
+
elsif @output.io.is_a?(RequestLogAnalyzer::Mailer)
|
362
357
|
@output.io.mail
|
363
358
|
end
|
364
359
|
end
|
365
360
|
|
366
361
|
def install_signal_handlers
|
367
|
-
Signal.trap(
|
362
|
+
Signal.trap('INT') do
|
368
363
|
handle_progress(:interrupted)
|
369
|
-
puts
|
364
|
+
puts 'Caught interrupt! Stopping parsing...'
|
370
365
|
@interrupted = true
|
371
366
|
end
|
372
367
|
end
|
373
|
-
|
374
368
|
end
|
375
369
|
end
|