request-log-analyzer 1.13.1 → 1.13.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/bin/console +17 -0
- data/lib/cli/command_line_arguments.rb +29 -36
- data/lib/cli/database_console.rb +1 -3
- data/lib/cli/database_console_init.rb +11 -11
- data/lib/cli/progressbar.rb +30 -32
- data/lib/cli/tools.rb +20 -23
- data/lib/request_log_analyzer.rb +8 -8
- data/lib/request_log_analyzer/aggregator.rb +4 -7
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +10 -13
- data/lib/request_log_analyzer/aggregator/echo.rb +5 -7
- data/lib/request_log_analyzer/aggregator/summarizer.rb +15 -18
- data/lib/request_log_analyzer/class_level_inheritable_attributes.rb +23 -0
- data/lib/request_log_analyzer/controller.rb +36 -42
- data/lib/request_log_analyzer/database.rb +4 -6
- data/lib/request_log_analyzer/database/base.rb +39 -41
- data/lib/request_log_analyzer/database/connection.rb +8 -10
- data/lib/request_log_analyzer/database/request.rb +1 -3
- data/lib/request_log_analyzer/database/source.rb +0 -2
- data/lib/request_log_analyzer/database/warning.rb +4 -6
- data/lib/request_log_analyzer/file_format.rb +46 -49
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +15 -19
- data/lib/request_log_analyzer/file_format/apache.rb +42 -45
- data/lib/request_log_analyzer/file_format/delayed_job.rb +13 -15
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job3.rb +5 -8
- data/lib/request_log_analyzer/file_format/delayed_job4.rb +5 -8
- data/lib/request_log_analyzer/file_format/haproxy.rb +44 -48
- data/lib/request_log_analyzer/file_format/merb.rb +13 -17
- data/lib/request_log_analyzer/file_format/mysql.rb +21 -25
- data/lib/request_log_analyzer/file_format/nginx.rb +0 -2
- data/lib/request_log_analyzer/file_format/oink.rb +30 -31
- data/lib/request_log_analyzer/file_format/postgresql.rb +11 -15
- data/lib/request_log_analyzer/file_format/rack.rb +0 -2
- data/lib/request_log_analyzer/file_format/rails.rb +100 -104
- data/lib/request_log_analyzer/file_format/rails3.rb +19 -23
- data/lib/request_log_analyzer/file_format/rails_development.rb +0 -1
- data/lib/request_log_analyzer/file_format/w3c.rb +16 -18
- data/lib/request_log_analyzer/filter.rb +0 -2
- data/lib/request_log_analyzer/filter/anonymize.rb +4 -7
- data/lib/request_log_analyzer/filter/field.rb +3 -6
- data/lib/request_log_analyzer/filter/timespan.rb +2 -6
- data/lib/request_log_analyzer/line_definition.rb +16 -19
- data/lib/request_log_analyzer/log_processor.rb +10 -14
- data/lib/request_log_analyzer/mailer.rb +9 -12
- data/lib/request_log_analyzer/output.rb +12 -14
- data/lib/request_log_analyzer/output/fixed_width.rb +21 -28
- data/lib/request_log_analyzer/output/html.rb +11 -14
- data/lib/request_log_analyzer/request.rb +53 -33
- data/lib/request_log_analyzer/source.rb +2 -5
- data/lib/request_log_analyzer/source/log_parser.rb +9 -16
- data/lib/request_log_analyzer/tracker.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +4 -6
- data/lib/request_log_analyzer/tracker/frequency.rb +9 -11
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -11
- data/lib/request_log_analyzer/tracker/numeric_value.rb +40 -44
- data/lib/request_log_analyzer/tracker/timespan.rb +5 -8
- data/lib/request_log_analyzer/tracker/traffic.rb +8 -10
- data/lib/request_log_analyzer/version.rb +1 -1
- data/request-log-analyzer.gemspec +6 -6
- data/spec/integration/command_line_usage_spec.rb +33 -33
- data/spec/integration/mailer_spec.rb +181 -185
- data/spec/integration/munin_plugins_rails_spec.rb +20 -20
- data/spec/integration/scout_spec.rb +40 -41
- data/spec/lib/helpers.rb +8 -9
- data/spec/lib/macros.rb +2 -4
- data/spec/lib/matchers.rb +20 -25
- data/spec/lib/mocks.rb +10 -11
- data/spec/lib/testing_format.rb +8 -10
- data/spec/spec_helper.rb +5 -1
- data/spec/unit/aggregator/database_inserter_spec.rb +23 -23
- data/spec/unit/aggregator/summarizer_spec.rb +7 -7
- data/spec/unit/controller/controller_spec.rb +14 -14
- data/spec/unit/controller/log_processor_spec.rb +3 -3
- data/spec/unit/database/base_class_spec.rb +36 -37
- data/spec/unit/database/connection_spec.rb +10 -10
- data/spec/unit/database/database_spec.rb +11 -11
- data/spec/unit/file_format/amazon_s3_format_spec.rb +66 -62
- data/spec/unit/file_format/apache_format_spec.rb +57 -52
- data/spec/unit/file_format/common_regular_expressions_spec.rb +18 -21
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job2_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job3_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job4_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job_format_spec.rb +12 -12
- data/spec/unit/file_format/file_format_api_spec.rb +19 -19
- data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
- data/spec/unit/file_format/haproxy_format_spec.rb +53 -49
- data/spec/unit/file_format/inheritance_spec.rb +13 -0
- data/spec/unit/file_format/line_definition_spec.rb +35 -33
- data/spec/unit/file_format/merb_format_spec.rb +13 -11
- data/spec/unit/file_format/mysql_format_spec.rb +24 -24
- data/spec/unit/file_format/oink_format_spec.rb +29 -29
- data/spec/unit/file_format/postgresql_format_spec.rb +9 -9
- data/spec/unit/file_format/rack_format_spec.rb +36 -31
- data/spec/unit/file_format/rails3_format_spec.rb +46 -46
- data/spec/unit/file_format/rails_format_spec.rb +52 -53
- data/spec/unit/file_format/w3c_format_spec.rb +27 -24
- data/spec/unit/filter/anonymize_filter_spec.rb +7 -7
- data/spec/unit/filter/field_filter_spec.rb +26 -26
- data/spec/unit/filter/filter_spec.rb +4 -4
- data/spec/unit/filter/timespan_filter_spec.rb +22 -22
- data/spec/unit/mailer_spec.rb +21 -21
- data/spec/unit/request_spec.rb +29 -29
- data/spec/unit/source/log_parser_spec.rb +5 -5
- data/spec/unit/tracker/duration_tracker_spec.rb +23 -23
- data/spec/unit/tracker/frequency_tracker_spec.rb +29 -30
- data/spec/unit/tracker/hourly_spread_spec.rb +35 -35
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +71 -72
- data/spec/unit/tracker/timespan_tracker_spec.rb +31 -31
- data/spec/unit/tracker/tracker_api_spec.rb +43 -44
- data/spec/unit/tracker/traffic_tracker_spec.rb +7 -7
- metadata +38 -35
@@ -2,7 +2,6 @@ require 'rubygems'
|
|
2
2
|
require 'active_record'
|
3
3
|
|
4
4
|
class RequestLogAnalyzer::Database
|
5
|
-
|
6
5
|
require 'request_log_analyzer/database/connection'
|
7
6
|
include RequestLogAnalyzer::Database::Connection
|
8
7
|
|
@@ -45,7 +44,6 @@ class RequestLogAnalyzer::Database
|
|
45
44
|
# Loads an ActiveRecord-based class that correspond to the given parameter, which can either be
|
46
45
|
# a table name or a LineDefinition instance.
|
47
46
|
def load_activerecord_class(linedefinition_or_table)
|
48
|
-
|
49
47
|
case linedefinition_or_table
|
50
48
|
when String, Symbol
|
51
49
|
klass_name = linedefinition_or_table.to_s.singularize.camelize
|
@@ -58,13 +56,13 @@ class RequestLogAnalyzer::Database
|
|
58
56
|
Object.const_set(klass_name, klass)
|
59
57
|
klass = Object.const_get(klass_name)
|
60
58
|
@line_classes << klass
|
61
|
-
|
59
|
+
klass
|
62
60
|
end
|
63
61
|
|
64
62
|
def fileformat_classes
|
65
|
-
|
66
|
-
line_classes = file_format.line_definitions.map { |(
|
67
|
-
|
63
|
+
fail 'No file_format provided!' unless file_format
|
64
|
+
line_classes = file_format.line_definitions.map { |(_name, definition)| load_activerecord_class(definition) }
|
65
|
+
default_classes + line_classes
|
68
66
|
end
|
69
67
|
|
70
68
|
# Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
|
@@ -1,5 +1,4 @@
|
|
1
1
|
class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
2
|
-
|
3
2
|
self.abstract_class = true
|
4
3
|
|
5
4
|
def <=>(other)
|
@@ -9,14 +8,14 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
9
8
|
source_comparison
|
10
9
|
end
|
11
10
|
end
|
12
|
-
|
11
|
+
|
13
12
|
# Handle format manually, because it is prohibidado in Rails 3.2.1
|
14
13
|
def format=(arg)
|
15
|
-
|
14
|
+
attributes[:format] = arg
|
16
15
|
end
|
17
|
-
|
18
|
-
def format(
|
19
|
-
|
16
|
+
|
17
|
+
def format(_arg)
|
18
|
+
attributes[:format]
|
20
19
|
end
|
21
20
|
|
22
21
|
def line_type
|
@@ -32,46 +31,46 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
32
31
|
klass.line_definition = definition
|
33
32
|
|
34
33
|
# Set relations with requests and sources table
|
35
|
-
klass.belongs_to :request, :
|
36
|
-
klass.belongs_to :source, :
|
34
|
+
klass.belongs_to :request, class_name: RequestLogAnalyzer::Database::Request.name
|
35
|
+
klass.belongs_to :source, class_name: RequestLogAnalyzer::Database::Source.name
|
37
36
|
|
38
37
|
# Serialize complex fields into the database
|
39
|
-
definition.captures.select { |c| c.
|
38
|
+
definition.captures.select { |c| c.key?(:provides) }.each do |capture|
|
40
39
|
klass.send(:serialize, capture[:name], Hash)
|
41
40
|
end
|
42
41
|
|
43
|
-
RequestLogAnalyzer::Database::Request.has_many
|
44
|
-
RequestLogAnalyzer::Database::Source.has_many
|
42
|
+
RequestLogAnalyzer::Database::Request.has_many "#{definition.name}_lines".to_sym
|
43
|
+
RequestLogAnalyzer::Database::Source.has_many "#{definition.name}_lines".to_sym
|
45
44
|
|
46
|
-
|
45
|
+
klass
|
47
46
|
end
|
48
47
|
|
49
48
|
def self.subclass_from_table(table, klass = Class.new(RequestLogAnalyzer::Database::Base))
|
50
|
-
|
51
|
-
|
49
|
+
fail "Table #{table} not found!" unless database.connection.table_exists?(table)
|
50
|
+
|
52
51
|
klass.table_name = table
|
53
52
|
|
54
53
|
if klass.column_names.include?('request_id')
|
55
|
-
klass.belongs_to :request, :
|
54
|
+
klass.belongs_to :request, class_name: RequestLogAnalyzer::Database::Request.name
|
56
55
|
RequestLogAnalyzer::Database::Request.has_many table.to_sym
|
57
56
|
end
|
58
57
|
|
59
58
|
if klass.column_names.include?('source_id')
|
60
|
-
klass.belongs_to :source, :
|
59
|
+
klass.belongs_to :source, class_name: RequestLogAnalyzer::Database::Source.name
|
61
60
|
RequestLogAnalyzer::Database::Source.has_many table.to_sym
|
62
61
|
end
|
63
62
|
|
64
|
-
|
63
|
+
klass
|
65
64
|
end
|
66
65
|
|
67
66
|
def self.drop_table!
|
68
|
-
database.connection.remove_index(
|
69
|
-
database.connection.remove_index(
|
70
|
-
database.connection.drop_table(
|
67
|
+
database.connection.remove_index(table_name, [:source_id]) rescue nil
|
68
|
+
database.connection.remove_index(table_name, [:request_id]) rescue nil
|
69
|
+
database.connection.drop_table(table_name) if database.connection.table_exists?(table_name)
|
71
70
|
end
|
72
71
|
|
73
72
|
def self.create_table!
|
74
|
-
|
73
|
+
fail 'No line_definition available to base table schema on!' unless line_definition
|
75
74
|
|
76
75
|
unless table_exists?
|
77
76
|
database.connection.create_table(table_name.to_sym) do |t|
|
@@ -88,13 +87,13 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
88
87
|
t.column(column_name, column_type(capture[:type]))
|
89
88
|
|
90
89
|
# If the capture provides other field as well, create columns for them, too
|
91
|
-
capture[:provides].each { |field, field_type| t.column(field, column_type(field_type)) } if capture[:provides].
|
90
|
+
capture[:provides].each { |field, field_type| t.column(field, column_type(field_type)) } if capture[:provides].is_a?(Hash)
|
92
91
|
end
|
93
92
|
end
|
94
|
-
|
93
|
+
|
95
94
|
# Add indices to table for more speedy querying
|
96
|
-
database.connection.add_index(
|
97
|
-
database.connection.add_index(
|
95
|
+
database.connection.add_index(table_name.to_sym, [:request_id]) # rescue
|
96
|
+
database.connection.add_index(table_name.to_sym, [:source_id]) # rescue
|
98
97
|
end
|
99
98
|
end
|
100
99
|
|
@@ -102,22 +101,21 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
102
101
|
# TODO: make more robust / include in file-format definition
|
103
102
|
def self.column_type(type_indicator)
|
104
103
|
case type_indicator
|
105
|
-
when :eval
|
106
|
-
when :hash
|
107
|
-
when :text
|
108
|
-
when :string
|
109
|
-
when :sec
|
110
|
-
when :msec
|
111
|
-
when :duration
|
112
|
-
when :float
|
113
|
-
when :double
|
114
|
-
when :integer
|
115
|
-
when :int
|
116
|
-
when :timestamp
|
117
|
-
when :datetime
|
118
|
-
when :date
|
104
|
+
when :eval then :text
|
105
|
+
when :hash then :text
|
106
|
+
when :text then :text
|
107
|
+
when :string then :string
|
108
|
+
when :sec then :float
|
109
|
+
when :msec then :float
|
110
|
+
when :duration then :float
|
111
|
+
when :float then :float
|
112
|
+
when :double then :float
|
113
|
+
when :integer then :integer
|
114
|
+
when :int then :int
|
115
|
+
when :timestamp then :datetime
|
116
|
+
when :datetime then :datetime
|
117
|
+
when :date then :date
|
119
118
|
else :string
|
120
119
|
end
|
121
120
|
end
|
122
|
-
|
123
|
-
end
|
121
|
+
end
|
@@ -1,29 +1,28 @@
|
|
1
1
|
module RequestLogAnalyzer::Database::Connection
|
2
|
-
|
3
2
|
def self.from_string(string)
|
4
3
|
hash = {}
|
5
4
|
if string =~ /^(?:\w+=(?:[^;])*;)*\w+=(?:[^;])*$/
|
6
5
|
string.scan(/(\w+)=([^;]*);?/) { |variable, value| hash[variable.to_sym] = value }
|
7
6
|
elsif string =~ /^(\w+)\:\/\/(?:(?:([^:]+)(?:\:([^:]+))?\@)?([\w\.-]+)\/)?([\w\:\-\.\/]+)$/
|
8
|
-
hash[:adapter], hash[:username], hash[:password], hash[:host], hash[:database] =
|
9
|
-
hash.delete_if { |
|
7
|
+
hash[:adapter], hash[:username], hash[:password], hash[:host], hash[:database] = Regexp.last_match[1], Regexp.last_match[2], Regexp.last_match[3], Regexp.last_match[4], Regexp.last_match[5]
|
8
|
+
hash.delete_if { |_k, v| v.nil? }
|
10
9
|
end
|
11
|
-
|
10
|
+
hash.empty? ? nil : hash
|
12
11
|
end
|
13
12
|
|
14
13
|
def connect(connection_identifier)
|
15
|
-
if connection_identifier.
|
14
|
+
if connection_identifier.is_a?(Hash)
|
16
15
|
ActiveRecord::Base.establish_connection(connection_identifier)
|
17
16
|
elsif connection_identifier == ':memory:'
|
18
|
-
ActiveRecord::Base.establish_connection(:
|
17
|
+
ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: ':memory:')
|
19
18
|
elsif connection_hash = RequestLogAnalyzer::Database::Connection.from_string(connection_identifier)
|
20
19
|
ActiveRecord::Base.establish_connection(connection_hash)
|
21
|
-
elsif connection_identifier.
|
22
|
-
ActiveRecord::Base.establish_connection(:
|
20
|
+
elsif connection_identifier.is_a?(String) # Normal SQLite 3 database file
|
21
|
+
ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: connection_identifier)
|
23
22
|
elsif connection_identifier.nil?
|
24
23
|
nil
|
25
24
|
else
|
26
|
-
|
25
|
+
fail "Cannot connect with this connection_identifier: #{connection_identifier.inspect}"
|
27
26
|
end
|
28
27
|
end
|
29
28
|
|
@@ -34,5 +33,4 @@ module RequestLogAnalyzer::Database::Connection
|
|
34
33
|
def connection
|
35
34
|
RequestLogAnalyzer::Database::Base.connection
|
36
35
|
end
|
37
|
-
|
38
36
|
end
|
@@ -1,10 +1,9 @@
|
|
1
1
|
class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base
|
2
|
-
|
3
2
|
# Returns an array of all the Line objects of this request in the correct order.
|
4
3
|
def lines
|
5
4
|
@lines ||= begin
|
6
5
|
lines = []
|
7
|
-
self.class.reflections.each { |r,
|
6
|
+
self.class.reflections.each { |r, _d| lines += send(r).all }
|
8
7
|
lines.sort
|
9
8
|
end
|
10
9
|
end
|
@@ -18,5 +17,4 @@ class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base
|
|
18
17
|
end
|
19
18
|
end
|
20
19
|
end
|
21
|
-
|
22
20
|
end
|
@@ -1,5 +1,4 @@
|
|
1
1
|
class RequestLogAnalyzer::Database::Source < RequestLogAnalyzer::Database::Base
|
2
|
-
|
3
2
|
def self.create_table!
|
4
3
|
unless database.connection.table_exists?(:sources)
|
5
4
|
database.connection.create_table(:sources) do |t|
|
@@ -9,5 +8,4 @@ class RequestLogAnalyzer::Database::Source < RequestLogAnalyzer::Database::Base
|
|
9
8
|
end
|
10
9
|
end
|
11
10
|
end
|
12
|
-
|
13
11
|
end
|
@@ -1,14 +1,12 @@
|
|
1
1
|
class RequestLogAnalyzer::Database::Warning < RequestLogAnalyzer::Database::Base
|
2
|
-
|
3
2
|
def self.create_table!
|
4
3
|
unless database.connection.table_exists?(:warnings)
|
5
4
|
database.connection.create_table(:warnings) do |t|
|
6
|
-
t.column
|
7
|
-
t.column
|
8
|
-
t.column
|
9
|
-
t.column
|
5
|
+
t.column :warning_type, :string, limit: 30, null: false
|
6
|
+
t.column :message, :string
|
7
|
+
t.column :source_id, :integer
|
8
|
+
t.column :lineno, :integer
|
10
9
|
end
|
11
10
|
end
|
12
11
|
end
|
13
|
-
|
14
12
|
end
|
@@ -1,7 +1,6 @@
|
|
1
1
|
require 'request_log_analyzer/request'
|
2
2
|
|
3
3
|
module RequestLogAnalyzer::FileFormat
|
4
|
-
|
5
4
|
autoload :Rails, 'request_log_analyzer/file_format/rails'
|
6
5
|
autoload :Rails3, 'request_log_analyzer/file_format/rails3'
|
7
6
|
autoload :RailsDevelopment, 'request_log_analyzer/file_format/rails_development'
|
@@ -29,15 +28,15 @@ module RequestLogAnalyzer::FileFormat
|
|
29
28
|
# * A symbol of a built-in file format (e.g. :rails)
|
30
29
|
def self.load(file_format, *args)
|
31
30
|
klass = nil
|
32
|
-
if file_format.
|
31
|
+
if file_format.is_a?(RequestLogAnalyzer::FileFormat::Base)
|
33
32
|
# this already is a file format! return itself
|
34
33
|
return @current_file_format = file_format
|
35
34
|
|
36
|
-
elsif file_format.
|
35
|
+
elsif file_format.is_a?(Class) && file_format.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
|
37
36
|
# a usable class is provided. Use this format class.
|
38
37
|
klass = file_format
|
39
38
|
|
40
|
-
elsif file_format.
|
39
|
+
elsif file_format.is_a?(String) && File.exist?(file_format) && File.file?(file_format)
|
41
40
|
# load a format from a ruby file
|
42
41
|
require File.expand_path(file_format)
|
43
42
|
|
@@ -47,7 +46,7 @@ module RequestLogAnalyzer::FileFormat
|
|
47
46
|
elsif Object.const_defined?(const)
|
48
47
|
klass = Object.const_get(const)
|
49
48
|
else
|
50
|
-
|
49
|
+
fail "Cannot load class #{const} from #{file_format}!"
|
51
50
|
end
|
52
51
|
|
53
52
|
else
|
@@ -56,19 +55,19 @@ module RequestLogAnalyzer::FileFormat
|
|
56
55
|
end
|
57
56
|
|
58
57
|
# check the returned klass to see if it can be used
|
59
|
-
|
60
|
-
|
58
|
+
fail "Could not load a file format from #{file_format.inspect}" if klass.nil?
|
59
|
+
fail "Invalid FileFormat class from #{file_format.inspect}" unless klass.is_a?(Class) && klass.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
|
61
60
|
|
62
61
|
@current_file_format = klass.create(*args) # return an instance of the class
|
63
62
|
end
|
64
|
-
|
63
|
+
|
65
64
|
# Returns an array of all FileFormat instances that are shipped with request-log-analyzer by default.
|
66
65
|
def self.all_formats
|
67
|
-
@all_formats ||= Dir[File.expand_path('file_format/*.rb', File.dirname(__FILE__))].map do |file|
|
68
|
-
|
66
|
+
@all_formats ||= Dir[File.expand_path('file_format/*.rb', File.dirname(__FILE__))].map do |file|
|
67
|
+
load(File.basename(file, '.rb'))
|
69
68
|
end
|
70
69
|
end
|
71
|
-
|
70
|
+
|
72
71
|
# Autodetects the filetype of a given file.
|
73
72
|
#
|
74
73
|
# Returns a FileFormat instance, by parsing the first couple of lines of the provided file
|
@@ -78,18 +77,17 @@ module RequestLogAnalyzer::FileFormat
|
|
78
77
|
# <tt>file</tt>:: The file to detect the file format for.
|
79
78
|
# <tt>line_count</tt>:: The number of lines to take into consideration
|
80
79
|
def self.autodetect(file, line_count = 50)
|
81
|
-
|
82
|
-
|
83
|
-
|
80
|
+
parsers = all_formats.map { |f| RequestLogAnalyzer::Source::LogParser.new(f, parse_strategy: 'cautious') }
|
81
|
+
|
84
82
|
File.open(file, 'rb') do |io|
|
85
83
|
while io.lineno < line_count && (line = io.gets)
|
86
|
-
parsers.each { |parser| parser.parse_line(line) }
|
84
|
+
parsers.each { |parser| parser.parse_line(line) }
|
87
85
|
end
|
88
86
|
end
|
89
|
-
|
87
|
+
|
90
88
|
parsers.select { |p| autodetect_score(p) > 0 }.max { |a, b| autodetect_score(a) <=> autodetect_score(b) }.file_format rescue nil
|
91
89
|
end
|
92
|
-
|
90
|
+
|
93
91
|
# Calculates a file format auto detection score based on the parser statistics.
|
94
92
|
#
|
95
93
|
# This method returns a score as an integer. Usually, the score will increase as more
|
@@ -113,10 +111,9 @@ module RequestLogAnalyzer::FileFormat
|
|
113
111
|
# This module contains some methods to construct regular expressions for log fragments
|
114
112
|
# that are commonly used, like IP addresses and timestamp.
|
115
113
|
#
|
116
|
-
# You need to extend (or include in an unlikely case) this module in your file format
|
114
|
+
# You need to extend (or include in an unlikely case) this module in your file format
|
117
115
|
# to use these regular expression constructors.
|
118
116
|
module CommonRegularExpressions
|
119
|
-
|
120
117
|
TIMESTAMP_PARTS = {
|
121
118
|
'a' => '(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)',
|
122
119
|
'b' => '(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)',
|
@@ -126,13 +123,13 @@ module RequestLogAnalyzer::FileFormat
|
|
126
123
|
'Z' => '(?:[+-]\d{4}|[A-Z]{3,4})',
|
127
124
|
'%' => '%'
|
128
125
|
}
|
129
|
-
|
126
|
+
|
130
127
|
# Creates a regular expression to match a hostname
|
131
128
|
def hostname(blank = false)
|
132
129
|
regexp = /(?:(?:[a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9])/
|
133
130
|
add_blank_option(regexp, blank)
|
134
131
|
end
|
135
|
-
|
132
|
+
|
136
133
|
# Creates a regular expression to match a hostname or ip address
|
137
134
|
def hostname_or_ip_address(blank = false)
|
138
135
|
regexp = Regexp.union(hostname, ip_address)
|
@@ -148,10 +145,10 @@ module RequestLogAnalyzer::FileFormat
|
|
148
145
|
format_string.scan(/([^%]*)(?:%([A-Za-z%]))?/) do |literal, variable|
|
149
146
|
regexp << Regexp.quote(literal)
|
150
147
|
if variable
|
151
|
-
if TIMESTAMP_PARTS.
|
148
|
+
if TIMESTAMP_PARTS.key?(variable)
|
152
149
|
regexp << TIMESTAMP_PARTS[variable]
|
153
150
|
else
|
154
|
-
|
151
|
+
fail "Unknown variable: %#{variable}"
|
155
152
|
end
|
156
153
|
end
|
157
154
|
end
|
@@ -161,11 +158,10 @@ module RequestLogAnalyzer::FileFormat
|
|
161
158
|
|
162
159
|
# Construct a regular expression to parse IPv4 and IPv6 addresses.
|
163
160
|
#
|
164
|
-
# Allow nil values if the blank option is given. This can be true to
|
161
|
+
# Allow nil values if the blank option is given. This can be true to
|
165
162
|
# allow an empty string or to a string substitute for the nil value.
|
166
163
|
def ip_address(blank = false)
|
167
|
-
|
168
|
-
# IP address regexp copied from Resolv::IPv4 and Resolv::IPv6,
|
164
|
+
# IP address regexp copied from Resolv::IPv4 and Resolv::IPv6,
|
169
165
|
# but adjusted to work for the purpose of request-log-analyzer.
|
170
166
|
ipv4_regexp = /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/
|
171
167
|
ipv6_regex_8_hex = /(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}/
|
@@ -176,19 +172,19 @@ module RequestLogAnalyzer::FileFormat
|
|
176
172
|
|
177
173
|
add_blank_option(Regexp.union(ipv4_regexp, ipv6_regexp), blank)
|
178
174
|
end
|
179
|
-
|
175
|
+
|
180
176
|
def anchored(regexp)
|
181
177
|
/^#{regexp}$/
|
182
178
|
end
|
183
|
-
|
179
|
+
|
184
180
|
protected
|
185
|
-
|
186
|
-
# Allow the field to be blank if this option is given. This can be true to
|
181
|
+
|
182
|
+
# Allow the field to be blank if this option is given. This can be true to
|
187
183
|
# allow an empty string or a string alternative for the nil value.
|
188
184
|
def add_blank_option(regexp, blank)
|
189
185
|
case blank
|
190
|
-
when String
|
191
|
-
when true
|
186
|
+
when String then Regexp.union(regexp, Regexp.new(Regexp.quote(blank)))
|
187
|
+
when true then Regexp.union(regexp, //)
|
192
188
|
else regexp
|
193
189
|
end
|
194
190
|
end
|
@@ -200,6 +196,8 @@ module RequestLogAnalyzer::FileFormat
|
|
200
196
|
# A subclass of this class is instantiated when request-log-analyzer is started and this instance
|
201
197
|
# is shared with all components of the application so they can act on the specifics of the format
|
202
198
|
class Base
|
199
|
+
extend RequestLogAnalyzer::ClassLevelInheritableAttributes
|
200
|
+
inheritable_attributes :line_definer, :report_definer
|
203
201
|
|
204
202
|
attr_reader :line_definitions, :report_trackers
|
205
203
|
|
@@ -213,18 +211,18 @@ module RequestLogAnalyzer::FileFormat
|
|
213
211
|
end
|
214
212
|
|
215
213
|
# Specifies multiple line definitions at once using a block
|
216
|
-
def self.format_definition(&
|
214
|
+
def self.format_definition(&_block)
|
217
215
|
if block_given?
|
218
|
-
yield
|
216
|
+
yield line_definer
|
219
217
|
else
|
220
|
-
return
|
218
|
+
return line_definer
|
221
219
|
end
|
222
220
|
end
|
223
221
|
|
224
222
|
# Specifies the summary report using a block.
|
225
|
-
def self.report(mode = :append, &
|
226
|
-
|
227
|
-
yield(
|
223
|
+
def self.report(mode = :append, &_block)
|
224
|
+
report_definer.reset! if mode == :overwrite
|
225
|
+
yield(report_definer)
|
228
226
|
end
|
229
227
|
|
230
228
|
# Setup the default line definer.
|
@@ -244,9 +242,9 @@ module RequestLogAnalyzer::FileFormat
|
|
244
242
|
# Instantiation
|
245
243
|
####################################################################################
|
246
244
|
|
247
|
-
def self.create(*
|
245
|
+
def self.create(*_args)
|
248
246
|
# Ignore arguments
|
249
|
-
|
247
|
+
new(line_definer.line_definitions, report_definer.trackers)
|
250
248
|
end
|
251
249
|
|
252
250
|
def initialize(line_definitions = {}, report_trackers = [])
|
@@ -271,16 +269,15 @@ module RequestLogAnalyzer::FileFormat
|
|
271
269
|
def well_formed?
|
272
270
|
valid_line_definitions? && valid_request_class?
|
273
271
|
end
|
274
|
-
|
272
|
+
|
275
273
|
alias_method :valid?, :well_formed?
|
276
|
-
|
277
274
|
|
278
275
|
# Checks whether the line definitions form a valid language.
|
279
|
-
# A file format should have at least a header and a footer line type
|
276
|
+
# A file format should have at least a header and a footer line type
|
280
277
|
def valid_line_definitions?
|
281
278
|
line_definitions.any? { |(_, ld)| ld.header } && line_definitions.any? { |(_, ld)| ld.footer }
|
282
279
|
end
|
283
|
-
|
280
|
+
|
284
281
|
# Checks whether the request class inherits from the base Request class.
|
285
282
|
def valid_request_class?
|
286
283
|
request_class.ancestors.include?(RequestLogAnalyzer::Request)
|
@@ -293,24 +290,24 @@ module RequestLogAnalyzer::FileFormat
|
|
293
290
|
|
294
291
|
# Function that a file format con implement to monkey patch the environment.
|
295
292
|
# * <tt>controller</tt> The environment is provided as a controller instance
|
296
|
-
def setup_environment(
|
293
|
+
def setup_environment(_controller)
|
297
294
|
end
|
298
295
|
|
299
296
|
# Parses a line by trying to parse it using every line definition in this file format
|
300
297
|
def parse_line(line, &warning_handler)
|
301
|
-
|
298
|
+
line_definitions.each do |_lt, definition|
|
302
299
|
match = definition.matches(line, &warning_handler)
|
303
300
|
return match if match
|
304
301
|
end
|
305
302
|
|
306
|
-
|
303
|
+
nil
|
307
304
|
end
|
308
|
-
|
305
|
+
|
309
306
|
# Returns the max line length for this file format if any.
|
310
307
|
def max_line_length
|
311
308
|
self.class.const_get(MAX_LINE_LENGTH) if self.class.const_defined?(:MAX_LINE_LENGTH)
|
312
309
|
end
|
313
|
-
|
310
|
+
|
314
311
|
def line_divider
|
315
312
|
self.class.const_get(LINE_DIVIDER) if self.class.const_defined?(:LINE_DIVIDER)
|
316
313
|
end
|