request-log-analyzer 1.13.1 → 1.13.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/bin/console +17 -0
- data/lib/cli/command_line_arguments.rb +29 -36
- data/lib/cli/database_console.rb +1 -3
- data/lib/cli/database_console_init.rb +11 -11
- data/lib/cli/progressbar.rb +30 -32
- data/lib/cli/tools.rb +20 -23
- data/lib/request_log_analyzer.rb +8 -8
- data/lib/request_log_analyzer/aggregator.rb +4 -7
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +10 -13
- data/lib/request_log_analyzer/aggregator/echo.rb +5 -7
- data/lib/request_log_analyzer/aggregator/summarizer.rb +15 -18
- data/lib/request_log_analyzer/class_level_inheritable_attributes.rb +23 -0
- data/lib/request_log_analyzer/controller.rb +36 -42
- data/lib/request_log_analyzer/database.rb +4 -6
- data/lib/request_log_analyzer/database/base.rb +39 -41
- data/lib/request_log_analyzer/database/connection.rb +8 -10
- data/lib/request_log_analyzer/database/request.rb +1 -3
- data/lib/request_log_analyzer/database/source.rb +0 -2
- data/lib/request_log_analyzer/database/warning.rb +4 -6
- data/lib/request_log_analyzer/file_format.rb +46 -49
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +15 -19
- data/lib/request_log_analyzer/file_format/apache.rb +42 -45
- data/lib/request_log_analyzer/file_format/delayed_job.rb +13 -15
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job3.rb +5 -8
- data/lib/request_log_analyzer/file_format/delayed_job4.rb +5 -8
- data/lib/request_log_analyzer/file_format/haproxy.rb +44 -48
- data/lib/request_log_analyzer/file_format/merb.rb +13 -17
- data/lib/request_log_analyzer/file_format/mysql.rb +21 -25
- data/lib/request_log_analyzer/file_format/nginx.rb +0 -2
- data/lib/request_log_analyzer/file_format/oink.rb +30 -31
- data/lib/request_log_analyzer/file_format/postgresql.rb +11 -15
- data/lib/request_log_analyzer/file_format/rack.rb +0 -2
- data/lib/request_log_analyzer/file_format/rails.rb +100 -104
- data/lib/request_log_analyzer/file_format/rails3.rb +19 -23
- data/lib/request_log_analyzer/file_format/rails_development.rb +0 -1
- data/lib/request_log_analyzer/file_format/w3c.rb +16 -18
- data/lib/request_log_analyzer/filter.rb +0 -2
- data/lib/request_log_analyzer/filter/anonymize.rb +4 -7
- data/lib/request_log_analyzer/filter/field.rb +3 -6
- data/lib/request_log_analyzer/filter/timespan.rb +2 -6
- data/lib/request_log_analyzer/line_definition.rb +16 -19
- data/lib/request_log_analyzer/log_processor.rb +10 -14
- data/lib/request_log_analyzer/mailer.rb +9 -12
- data/lib/request_log_analyzer/output.rb +12 -14
- data/lib/request_log_analyzer/output/fixed_width.rb +21 -28
- data/lib/request_log_analyzer/output/html.rb +11 -14
- data/lib/request_log_analyzer/request.rb +53 -33
- data/lib/request_log_analyzer/source.rb +2 -5
- data/lib/request_log_analyzer/source/log_parser.rb +9 -16
- data/lib/request_log_analyzer/tracker.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +4 -6
- data/lib/request_log_analyzer/tracker/frequency.rb +9 -11
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -11
- data/lib/request_log_analyzer/tracker/numeric_value.rb +40 -44
- data/lib/request_log_analyzer/tracker/timespan.rb +5 -8
- data/lib/request_log_analyzer/tracker/traffic.rb +8 -10
- data/lib/request_log_analyzer/version.rb +1 -1
- data/request-log-analyzer.gemspec +6 -6
- data/spec/integration/command_line_usage_spec.rb +33 -33
- data/spec/integration/mailer_spec.rb +181 -185
- data/spec/integration/munin_plugins_rails_spec.rb +20 -20
- data/spec/integration/scout_spec.rb +40 -41
- data/spec/lib/helpers.rb +8 -9
- data/spec/lib/macros.rb +2 -4
- data/spec/lib/matchers.rb +20 -25
- data/spec/lib/mocks.rb +10 -11
- data/spec/lib/testing_format.rb +8 -10
- data/spec/spec_helper.rb +5 -1
- data/spec/unit/aggregator/database_inserter_spec.rb +23 -23
- data/spec/unit/aggregator/summarizer_spec.rb +7 -7
- data/spec/unit/controller/controller_spec.rb +14 -14
- data/spec/unit/controller/log_processor_spec.rb +3 -3
- data/spec/unit/database/base_class_spec.rb +36 -37
- data/spec/unit/database/connection_spec.rb +10 -10
- data/spec/unit/database/database_spec.rb +11 -11
- data/spec/unit/file_format/amazon_s3_format_spec.rb +66 -62
- data/spec/unit/file_format/apache_format_spec.rb +57 -52
- data/spec/unit/file_format/common_regular_expressions_spec.rb +18 -21
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job2_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job3_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job4_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job_format_spec.rb +12 -12
- data/spec/unit/file_format/file_format_api_spec.rb +19 -19
- data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
- data/spec/unit/file_format/haproxy_format_spec.rb +53 -49
- data/spec/unit/file_format/inheritance_spec.rb +13 -0
- data/spec/unit/file_format/line_definition_spec.rb +35 -33
- data/spec/unit/file_format/merb_format_spec.rb +13 -11
- data/spec/unit/file_format/mysql_format_spec.rb +24 -24
- data/spec/unit/file_format/oink_format_spec.rb +29 -29
- data/spec/unit/file_format/postgresql_format_spec.rb +9 -9
- data/spec/unit/file_format/rack_format_spec.rb +36 -31
- data/spec/unit/file_format/rails3_format_spec.rb +46 -46
- data/spec/unit/file_format/rails_format_spec.rb +52 -53
- data/spec/unit/file_format/w3c_format_spec.rb +27 -24
- data/spec/unit/filter/anonymize_filter_spec.rb +7 -7
- data/spec/unit/filter/field_filter_spec.rb +26 -26
- data/spec/unit/filter/filter_spec.rb +4 -4
- data/spec/unit/filter/timespan_filter_spec.rb +22 -22
- data/spec/unit/mailer_spec.rb +21 -21
- data/spec/unit/request_spec.rb +29 -29
- data/spec/unit/source/log_parser_spec.rb +5 -5
- data/spec/unit/tracker/duration_tracker_spec.rb +23 -23
- data/spec/unit/tracker/frequency_tracker_spec.rb +29 -30
- data/spec/unit/tracker/hourly_spread_spec.rb +35 -35
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +71 -72
- data/spec/unit/tracker/timespan_tracker_spec.rb +31 -31
- data/spec/unit/tracker/tracker_api_spec.rb +43 -44
- data/spec/unit/tracker/traffic_tracker_spec.rb +7 -7
- metadata +38 -35
|
@@ -2,7 +2,6 @@ require 'rubygems'
|
|
|
2
2
|
require 'active_record'
|
|
3
3
|
|
|
4
4
|
class RequestLogAnalyzer::Database
|
|
5
|
-
|
|
6
5
|
require 'request_log_analyzer/database/connection'
|
|
7
6
|
include RequestLogAnalyzer::Database::Connection
|
|
8
7
|
|
|
@@ -45,7 +44,6 @@ class RequestLogAnalyzer::Database
|
|
|
45
44
|
# Loads an ActiveRecord-based class that correspond to the given parameter, which can either be
|
|
46
45
|
# a table name or a LineDefinition instance.
|
|
47
46
|
def load_activerecord_class(linedefinition_or_table)
|
|
48
|
-
|
|
49
47
|
case linedefinition_or_table
|
|
50
48
|
when String, Symbol
|
|
51
49
|
klass_name = linedefinition_or_table.to_s.singularize.camelize
|
|
@@ -58,13 +56,13 @@ class RequestLogAnalyzer::Database
|
|
|
58
56
|
Object.const_set(klass_name, klass)
|
|
59
57
|
klass = Object.const_get(klass_name)
|
|
60
58
|
@line_classes << klass
|
|
61
|
-
|
|
59
|
+
klass
|
|
62
60
|
end
|
|
63
61
|
|
|
64
62
|
def fileformat_classes
|
|
65
|
-
|
|
66
|
-
line_classes = file_format.line_definitions.map { |(
|
|
67
|
-
|
|
63
|
+
fail 'No file_format provided!' unless file_format
|
|
64
|
+
line_classes = file_format.line_definitions.map { |(_name, definition)| load_activerecord_class(definition) }
|
|
65
|
+
default_classes + line_classes
|
|
68
66
|
end
|
|
69
67
|
|
|
70
68
|
# Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
2
|
-
|
|
3
2
|
self.abstract_class = true
|
|
4
3
|
|
|
5
4
|
def <=>(other)
|
|
@@ -9,14 +8,14 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
|
9
8
|
source_comparison
|
|
10
9
|
end
|
|
11
10
|
end
|
|
12
|
-
|
|
11
|
+
|
|
13
12
|
# Handle format manually, because it is prohibidado in Rails 3.2.1
|
|
14
13
|
def format=(arg)
|
|
15
|
-
|
|
14
|
+
attributes[:format] = arg
|
|
16
15
|
end
|
|
17
|
-
|
|
18
|
-
def format(
|
|
19
|
-
|
|
16
|
+
|
|
17
|
+
def format(_arg)
|
|
18
|
+
attributes[:format]
|
|
20
19
|
end
|
|
21
20
|
|
|
22
21
|
def line_type
|
|
@@ -32,46 +31,46 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
|
32
31
|
klass.line_definition = definition
|
|
33
32
|
|
|
34
33
|
# Set relations with requests and sources table
|
|
35
|
-
klass.belongs_to :request, :
|
|
36
|
-
klass.belongs_to :source, :
|
|
34
|
+
klass.belongs_to :request, class_name: RequestLogAnalyzer::Database::Request.name
|
|
35
|
+
klass.belongs_to :source, class_name: RequestLogAnalyzer::Database::Source.name
|
|
37
36
|
|
|
38
37
|
# Serialize complex fields into the database
|
|
39
|
-
definition.captures.select { |c| c.
|
|
38
|
+
definition.captures.select { |c| c.key?(:provides) }.each do |capture|
|
|
40
39
|
klass.send(:serialize, capture[:name], Hash)
|
|
41
40
|
end
|
|
42
41
|
|
|
43
|
-
RequestLogAnalyzer::Database::Request.has_many
|
|
44
|
-
RequestLogAnalyzer::Database::Source.has_many
|
|
42
|
+
RequestLogAnalyzer::Database::Request.has_many "#{definition.name}_lines".to_sym
|
|
43
|
+
RequestLogAnalyzer::Database::Source.has_many "#{definition.name}_lines".to_sym
|
|
45
44
|
|
|
46
|
-
|
|
45
|
+
klass
|
|
47
46
|
end
|
|
48
47
|
|
|
49
48
|
def self.subclass_from_table(table, klass = Class.new(RequestLogAnalyzer::Database::Base))
|
|
50
|
-
|
|
51
|
-
|
|
49
|
+
fail "Table #{table} not found!" unless database.connection.table_exists?(table)
|
|
50
|
+
|
|
52
51
|
klass.table_name = table
|
|
53
52
|
|
|
54
53
|
if klass.column_names.include?('request_id')
|
|
55
|
-
klass.belongs_to :request, :
|
|
54
|
+
klass.belongs_to :request, class_name: RequestLogAnalyzer::Database::Request.name
|
|
56
55
|
RequestLogAnalyzer::Database::Request.has_many table.to_sym
|
|
57
56
|
end
|
|
58
57
|
|
|
59
58
|
if klass.column_names.include?('source_id')
|
|
60
|
-
klass.belongs_to :source, :
|
|
59
|
+
klass.belongs_to :source, class_name: RequestLogAnalyzer::Database::Source.name
|
|
61
60
|
RequestLogAnalyzer::Database::Source.has_many table.to_sym
|
|
62
61
|
end
|
|
63
62
|
|
|
64
|
-
|
|
63
|
+
klass
|
|
65
64
|
end
|
|
66
65
|
|
|
67
66
|
def self.drop_table!
|
|
68
|
-
database.connection.remove_index(
|
|
69
|
-
database.connection.remove_index(
|
|
70
|
-
database.connection.drop_table(
|
|
67
|
+
database.connection.remove_index(table_name, [:source_id]) rescue nil
|
|
68
|
+
database.connection.remove_index(table_name, [:request_id]) rescue nil
|
|
69
|
+
database.connection.drop_table(table_name) if database.connection.table_exists?(table_name)
|
|
71
70
|
end
|
|
72
71
|
|
|
73
72
|
def self.create_table!
|
|
74
|
-
|
|
73
|
+
fail 'No line_definition available to base table schema on!' unless line_definition
|
|
75
74
|
|
|
76
75
|
unless table_exists?
|
|
77
76
|
database.connection.create_table(table_name.to_sym) do |t|
|
|
@@ -88,13 +87,13 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
|
88
87
|
t.column(column_name, column_type(capture[:type]))
|
|
89
88
|
|
|
90
89
|
# If the capture provides other field as well, create columns for them, too
|
|
91
|
-
capture[:provides].each { |field, field_type| t.column(field, column_type(field_type)) } if capture[:provides].
|
|
90
|
+
capture[:provides].each { |field, field_type| t.column(field, column_type(field_type)) } if capture[:provides].is_a?(Hash)
|
|
92
91
|
end
|
|
93
92
|
end
|
|
94
|
-
|
|
93
|
+
|
|
95
94
|
# Add indices to table for more speedy querying
|
|
96
|
-
database.connection.add_index(
|
|
97
|
-
database.connection.add_index(
|
|
95
|
+
database.connection.add_index(table_name.to_sym, [:request_id]) # rescue
|
|
96
|
+
database.connection.add_index(table_name.to_sym, [:source_id]) # rescue
|
|
98
97
|
end
|
|
99
98
|
end
|
|
100
99
|
|
|
@@ -102,22 +101,21 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
|
102
101
|
# TODO: make more robust / include in file-format definition
|
|
103
102
|
def self.column_type(type_indicator)
|
|
104
103
|
case type_indicator
|
|
105
|
-
when :eval
|
|
106
|
-
when :hash
|
|
107
|
-
when :text
|
|
108
|
-
when :string
|
|
109
|
-
when :sec
|
|
110
|
-
when :msec
|
|
111
|
-
when :duration
|
|
112
|
-
when :float
|
|
113
|
-
when :double
|
|
114
|
-
when :integer
|
|
115
|
-
when :int
|
|
116
|
-
when :timestamp
|
|
117
|
-
when :datetime
|
|
118
|
-
when :date
|
|
104
|
+
when :eval then :text
|
|
105
|
+
when :hash then :text
|
|
106
|
+
when :text then :text
|
|
107
|
+
when :string then :string
|
|
108
|
+
when :sec then :float
|
|
109
|
+
when :msec then :float
|
|
110
|
+
when :duration then :float
|
|
111
|
+
when :float then :float
|
|
112
|
+
when :double then :float
|
|
113
|
+
when :integer then :integer
|
|
114
|
+
when :int then :int
|
|
115
|
+
when :timestamp then :datetime
|
|
116
|
+
when :datetime then :datetime
|
|
117
|
+
when :date then :date
|
|
119
118
|
else :string
|
|
120
119
|
end
|
|
121
120
|
end
|
|
122
|
-
|
|
123
|
-
end
|
|
121
|
+
end
|
|
@@ -1,29 +1,28 @@
|
|
|
1
1
|
module RequestLogAnalyzer::Database::Connection
|
|
2
|
-
|
|
3
2
|
def self.from_string(string)
|
|
4
3
|
hash = {}
|
|
5
4
|
if string =~ /^(?:\w+=(?:[^;])*;)*\w+=(?:[^;])*$/
|
|
6
5
|
string.scan(/(\w+)=([^;]*);?/) { |variable, value| hash[variable.to_sym] = value }
|
|
7
6
|
elsif string =~ /^(\w+)\:\/\/(?:(?:([^:]+)(?:\:([^:]+))?\@)?([\w\.-]+)\/)?([\w\:\-\.\/]+)$/
|
|
8
|
-
hash[:adapter], hash[:username], hash[:password], hash[:host], hash[:database] =
|
|
9
|
-
hash.delete_if { |
|
|
7
|
+
hash[:adapter], hash[:username], hash[:password], hash[:host], hash[:database] = Regexp.last_match[1], Regexp.last_match[2], Regexp.last_match[3], Regexp.last_match[4], Regexp.last_match[5]
|
|
8
|
+
hash.delete_if { |_k, v| v.nil? }
|
|
10
9
|
end
|
|
11
|
-
|
|
10
|
+
hash.empty? ? nil : hash
|
|
12
11
|
end
|
|
13
12
|
|
|
14
13
|
def connect(connection_identifier)
|
|
15
|
-
if connection_identifier.
|
|
14
|
+
if connection_identifier.is_a?(Hash)
|
|
16
15
|
ActiveRecord::Base.establish_connection(connection_identifier)
|
|
17
16
|
elsif connection_identifier == ':memory:'
|
|
18
|
-
ActiveRecord::Base.establish_connection(:
|
|
17
|
+
ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: ':memory:')
|
|
19
18
|
elsif connection_hash = RequestLogAnalyzer::Database::Connection.from_string(connection_identifier)
|
|
20
19
|
ActiveRecord::Base.establish_connection(connection_hash)
|
|
21
|
-
elsif connection_identifier.
|
|
22
|
-
ActiveRecord::Base.establish_connection(:
|
|
20
|
+
elsif connection_identifier.is_a?(String) # Normal SQLite 3 database file
|
|
21
|
+
ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: connection_identifier)
|
|
23
22
|
elsif connection_identifier.nil?
|
|
24
23
|
nil
|
|
25
24
|
else
|
|
26
|
-
|
|
25
|
+
fail "Cannot connect with this connection_identifier: #{connection_identifier.inspect}"
|
|
27
26
|
end
|
|
28
27
|
end
|
|
29
28
|
|
|
@@ -34,5 +33,4 @@ module RequestLogAnalyzer::Database::Connection
|
|
|
34
33
|
def connection
|
|
35
34
|
RequestLogAnalyzer::Database::Base.connection
|
|
36
35
|
end
|
|
37
|
-
|
|
38
36
|
end
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base
|
|
2
|
-
|
|
3
2
|
# Returns an array of all the Line objects of this request in the correct order.
|
|
4
3
|
def lines
|
|
5
4
|
@lines ||= begin
|
|
6
5
|
lines = []
|
|
7
|
-
self.class.reflections.each { |r,
|
|
6
|
+
self.class.reflections.each { |r, _d| lines += send(r).all }
|
|
8
7
|
lines.sort
|
|
9
8
|
end
|
|
10
9
|
end
|
|
@@ -18,5 +17,4 @@ class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base
|
|
|
18
17
|
end
|
|
19
18
|
end
|
|
20
19
|
end
|
|
21
|
-
|
|
22
20
|
end
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
class RequestLogAnalyzer::Database::Source < RequestLogAnalyzer::Database::Base
|
|
2
|
-
|
|
3
2
|
def self.create_table!
|
|
4
3
|
unless database.connection.table_exists?(:sources)
|
|
5
4
|
database.connection.create_table(:sources) do |t|
|
|
@@ -9,5 +8,4 @@ class RequestLogAnalyzer::Database::Source < RequestLogAnalyzer::Database::Base
|
|
|
9
8
|
end
|
|
10
9
|
end
|
|
11
10
|
end
|
|
12
|
-
|
|
13
11
|
end
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
class RequestLogAnalyzer::Database::Warning < RequestLogAnalyzer::Database::Base
|
|
2
|
-
|
|
3
2
|
def self.create_table!
|
|
4
3
|
unless database.connection.table_exists?(:warnings)
|
|
5
4
|
database.connection.create_table(:warnings) do |t|
|
|
6
|
-
t.column
|
|
7
|
-
t.column
|
|
8
|
-
t.column
|
|
9
|
-
t.column
|
|
5
|
+
t.column :warning_type, :string, limit: 30, null: false
|
|
6
|
+
t.column :message, :string
|
|
7
|
+
t.column :source_id, :integer
|
|
8
|
+
t.column :lineno, :integer
|
|
10
9
|
end
|
|
11
10
|
end
|
|
12
11
|
end
|
|
13
|
-
|
|
14
12
|
end
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
require 'request_log_analyzer/request'
|
|
2
2
|
|
|
3
3
|
module RequestLogAnalyzer::FileFormat
|
|
4
|
-
|
|
5
4
|
autoload :Rails, 'request_log_analyzer/file_format/rails'
|
|
6
5
|
autoload :Rails3, 'request_log_analyzer/file_format/rails3'
|
|
7
6
|
autoload :RailsDevelopment, 'request_log_analyzer/file_format/rails_development'
|
|
@@ -29,15 +28,15 @@ module RequestLogAnalyzer::FileFormat
|
|
|
29
28
|
# * A symbol of a built-in file format (e.g. :rails)
|
|
30
29
|
def self.load(file_format, *args)
|
|
31
30
|
klass = nil
|
|
32
|
-
if file_format.
|
|
31
|
+
if file_format.is_a?(RequestLogAnalyzer::FileFormat::Base)
|
|
33
32
|
# this already is a file format! return itself
|
|
34
33
|
return @current_file_format = file_format
|
|
35
34
|
|
|
36
|
-
elsif file_format.
|
|
35
|
+
elsif file_format.is_a?(Class) && file_format.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
|
|
37
36
|
# a usable class is provided. Use this format class.
|
|
38
37
|
klass = file_format
|
|
39
38
|
|
|
40
|
-
elsif file_format.
|
|
39
|
+
elsif file_format.is_a?(String) && File.exist?(file_format) && File.file?(file_format)
|
|
41
40
|
# load a format from a ruby file
|
|
42
41
|
require File.expand_path(file_format)
|
|
43
42
|
|
|
@@ -47,7 +46,7 @@ module RequestLogAnalyzer::FileFormat
|
|
|
47
46
|
elsif Object.const_defined?(const)
|
|
48
47
|
klass = Object.const_get(const)
|
|
49
48
|
else
|
|
50
|
-
|
|
49
|
+
fail "Cannot load class #{const} from #{file_format}!"
|
|
51
50
|
end
|
|
52
51
|
|
|
53
52
|
else
|
|
@@ -56,19 +55,19 @@ module RequestLogAnalyzer::FileFormat
|
|
|
56
55
|
end
|
|
57
56
|
|
|
58
57
|
# check the returned klass to see if it can be used
|
|
59
|
-
|
|
60
|
-
|
|
58
|
+
fail "Could not load a file format from #{file_format.inspect}" if klass.nil?
|
|
59
|
+
fail "Invalid FileFormat class from #{file_format.inspect}" unless klass.is_a?(Class) && klass.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
|
|
61
60
|
|
|
62
61
|
@current_file_format = klass.create(*args) # return an instance of the class
|
|
63
62
|
end
|
|
64
|
-
|
|
63
|
+
|
|
65
64
|
# Returns an array of all FileFormat instances that are shipped with request-log-analyzer by default.
|
|
66
65
|
def self.all_formats
|
|
67
|
-
@all_formats ||= Dir[File.expand_path('file_format/*.rb', File.dirname(__FILE__))].map do |file|
|
|
68
|
-
|
|
66
|
+
@all_formats ||= Dir[File.expand_path('file_format/*.rb', File.dirname(__FILE__))].map do |file|
|
|
67
|
+
load(File.basename(file, '.rb'))
|
|
69
68
|
end
|
|
70
69
|
end
|
|
71
|
-
|
|
70
|
+
|
|
72
71
|
# Autodetects the filetype of a given file.
|
|
73
72
|
#
|
|
74
73
|
# Returns a FileFormat instance, by parsing the first couple of lines of the provided file
|
|
@@ -78,18 +77,17 @@ module RequestLogAnalyzer::FileFormat
|
|
|
78
77
|
# <tt>file</tt>:: The file to detect the file format for.
|
|
79
78
|
# <tt>line_count</tt>:: The number of lines to take into consideration
|
|
80
79
|
def self.autodetect(file, line_count = 50)
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
80
|
+
parsers = all_formats.map { |f| RequestLogAnalyzer::Source::LogParser.new(f, parse_strategy: 'cautious') }
|
|
81
|
+
|
|
84
82
|
File.open(file, 'rb') do |io|
|
|
85
83
|
while io.lineno < line_count && (line = io.gets)
|
|
86
|
-
parsers.each { |parser| parser.parse_line(line) }
|
|
84
|
+
parsers.each { |parser| parser.parse_line(line) }
|
|
87
85
|
end
|
|
88
86
|
end
|
|
89
|
-
|
|
87
|
+
|
|
90
88
|
parsers.select { |p| autodetect_score(p) > 0 }.max { |a, b| autodetect_score(a) <=> autodetect_score(b) }.file_format rescue nil
|
|
91
89
|
end
|
|
92
|
-
|
|
90
|
+
|
|
93
91
|
# Calculates a file format auto detection score based on the parser statistics.
|
|
94
92
|
#
|
|
95
93
|
# This method returns a score as an integer. Usually, the score will increase as more
|
|
@@ -113,10 +111,9 @@ module RequestLogAnalyzer::FileFormat
|
|
|
113
111
|
# This module contains some methods to construct regular expressions for log fragments
|
|
114
112
|
# that are commonly used, like IP addresses and timestamp.
|
|
115
113
|
#
|
|
116
|
-
# You need to extend (or include in an unlikely case) this module in your file format
|
|
114
|
+
# You need to extend (or include in an unlikely case) this module in your file format
|
|
117
115
|
# to use these regular expression constructors.
|
|
118
116
|
module CommonRegularExpressions
|
|
119
|
-
|
|
120
117
|
TIMESTAMP_PARTS = {
|
|
121
118
|
'a' => '(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)',
|
|
122
119
|
'b' => '(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)',
|
|
@@ -126,13 +123,13 @@ module RequestLogAnalyzer::FileFormat
|
|
|
126
123
|
'Z' => '(?:[+-]\d{4}|[A-Z]{3,4})',
|
|
127
124
|
'%' => '%'
|
|
128
125
|
}
|
|
129
|
-
|
|
126
|
+
|
|
130
127
|
# Creates a regular expression to match a hostname
|
|
131
128
|
def hostname(blank = false)
|
|
132
129
|
regexp = /(?:(?:[a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9])/
|
|
133
130
|
add_blank_option(regexp, blank)
|
|
134
131
|
end
|
|
135
|
-
|
|
132
|
+
|
|
136
133
|
# Creates a regular expression to match a hostname or ip address
|
|
137
134
|
def hostname_or_ip_address(blank = false)
|
|
138
135
|
regexp = Regexp.union(hostname, ip_address)
|
|
@@ -148,10 +145,10 @@ module RequestLogAnalyzer::FileFormat
|
|
|
148
145
|
format_string.scan(/([^%]*)(?:%([A-Za-z%]))?/) do |literal, variable|
|
|
149
146
|
regexp << Regexp.quote(literal)
|
|
150
147
|
if variable
|
|
151
|
-
if TIMESTAMP_PARTS.
|
|
148
|
+
if TIMESTAMP_PARTS.key?(variable)
|
|
152
149
|
regexp << TIMESTAMP_PARTS[variable]
|
|
153
150
|
else
|
|
154
|
-
|
|
151
|
+
fail "Unknown variable: %#{variable}"
|
|
155
152
|
end
|
|
156
153
|
end
|
|
157
154
|
end
|
|
@@ -161,11 +158,10 @@ module RequestLogAnalyzer::FileFormat
|
|
|
161
158
|
|
|
162
159
|
# Construct a regular expression to parse IPv4 and IPv6 addresses.
|
|
163
160
|
#
|
|
164
|
-
# Allow nil values if the blank option is given. This can be true to
|
|
161
|
+
# Allow nil values if the blank option is given. This can be true to
|
|
165
162
|
# allow an empty string or to a string substitute for the nil value.
|
|
166
163
|
def ip_address(blank = false)
|
|
167
|
-
|
|
168
|
-
# IP address regexp copied from Resolv::IPv4 and Resolv::IPv6,
|
|
164
|
+
# IP address regexp copied from Resolv::IPv4 and Resolv::IPv6,
|
|
169
165
|
# but adjusted to work for the purpose of request-log-analyzer.
|
|
170
166
|
ipv4_regexp = /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/
|
|
171
167
|
ipv6_regex_8_hex = /(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}/
|
|
@@ -176,19 +172,19 @@ module RequestLogAnalyzer::FileFormat
|
|
|
176
172
|
|
|
177
173
|
add_blank_option(Regexp.union(ipv4_regexp, ipv6_regexp), blank)
|
|
178
174
|
end
|
|
179
|
-
|
|
175
|
+
|
|
180
176
|
def anchored(regexp)
|
|
181
177
|
/^#{regexp}$/
|
|
182
178
|
end
|
|
183
|
-
|
|
179
|
+
|
|
184
180
|
protected
|
|
185
|
-
|
|
186
|
-
# Allow the field to be blank if this option is given. This can be true to
|
|
181
|
+
|
|
182
|
+
# Allow the field to be blank if this option is given. This can be true to
|
|
187
183
|
# allow an empty string or a string alternative for the nil value.
|
|
188
184
|
def add_blank_option(regexp, blank)
|
|
189
185
|
case blank
|
|
190
|
-
when String
|
|
191
|
-
when true
|
|
186
|
+
when String then Regexp.union(regexp, Regexp.new(Regexp.quote(blank)))
|
|
187
|
+
when true then Regexp.union(regexp, //)
|
|
192
188
|
else regexp
|
|
193
189
|
end
|
|
194
190
|
end
|
|
@@ -200,6 +196,8 @@ module RequestLogAnalyzer::FileFormat
|
|
|
200
196
|
# A subclass of this class is instantiated when request-log-analyzer is started and this instance
|
|
201
197
|
# is shared with all components of the application so they can act on the specifics of the format
|
|
202
198
|
class Base
|
|
199
|
+
extend RequestLogAnalyzer::ClassLevelInheritableAttributes
|
|
200
|
+
inheritable_attributes :line_definer, :report_definer
|
|
203
201
|
|
|
204
202
|
attr_reader :line_definitions, :report_trackers
|
|
205
203
|
|
|
@@ -213,18 +211,18 @@ module RequestLogAnalyzer::FileFormat
|
|
|
213
211
|
end
|
|
214
212
|
|
|
215
213
|
# Specifies multiple line definitions at once using a block
|
|
216
|
-
def self.format_definition(&
|
|
214
|
+
def self.format_definition(&_block)
|
|
217
215
|
if block_given?
|
|
218
|
-
yield
|
|
216
|
+
yield line_definer
|
|
219
217
|
else
|
|
220
|
-
return
|
|
218
|
+
return line_definer
|
|
221
219
|
end
|
|
222
220
|
end
|
|
223
221
|
|
|
224
222
|
# Specifies the summary report using a block.
|
|
225
|
-
def self.report(mode = :append, &
|
|
226
|
-
|
|
227
|
-
yield(
|
|
223
|
+
def self.report(mode = :append, &_block)
|
|
224
|
+
report_definer.reset! if mode == :overwrite
|
|
225
|
+
yield(report_definer)
|
|
228
226
|
end
|
|
229
227
|
|
|
230
228
|
# Setup the default line definer.
|
|
@@ -244,9 +242,9 @@ module RequestLogAnalyzer::FileFormat
|
|
|
244
242
|
# Instantiation
|
|
245
243
|
####################################################################################
|
|
246
244
|
|
|
247
|
-
def self.create(*
|
|
245
|
+
def self.create(*_args)
|
|
248
246
|
# Ignore arguments
|
|
249
|
-
|
|
247
|
+
new(line_definer.line_definitions, report_definer.trackers)
|
|
250
248
|
end
|
|
251
249
|
|
|
252
250
|
def initialize(line_definitions = {}, report_trackers = [])
|
|
@@ -271,16 +269,15 @@ module RequestLogAnalyzer::FileFormat
|
|
|
271
269
|
def well_formed?
|
|
272
270
|
valid_line_definitions? && valid_request_class?
|
|
273
271
|
end
|
|
274
|
-
|
|
272
|
+
|
|
275
273
|
alias_method :valid?, :well_formed?
|
|
276
|
-
|
|
277
274
|
|
|
278
275
|
# Checks whether the line definitions form a valid language.
|
|
279
|
-
# A file format should have at least a header and a footer line type
|
|
276
|
+
# A file format should have at least a header and a footer line type
|
|
280
277
|
def valid_line_definitions?
|
|
281
278
|
line_definitions.any? { |(_, ld)| ld.header } && line_definitions.any? { |(_, ld)| ld.footer }
|
|
282
279
|
end
|
|
283
|
-
|
|
280
|
+
|
|
284
281
|
# Checks whether the request class inherits from the base Request class.
|
|
285
282
|
def valid_request_class?
|
|
286
283
|
request_class.ancestors.include?(RequestLogAnalyzer::Request)
|
|
@@ -293,24 +290,24 @@ module RequestLogAnalyzer::FileFormat
|
|
|
293
290
|
|
|
294
291
|
# Function that a file format con implement to monkey patch the environment.
|
|
295
292
|
# * <tt>controller</tt> The environment is provided as a controller instance
|
|
296
|
-
def setup_environment(
|
|
293
|
+
def setup_environment(_controller)
|
|
297
294
|
end
|
|
298
295
|
|
|
299
296
|
# Parses a line by trying to parse it using every line definition in this file format
|
|
300
297
|
def parse_line(line, &warning_handler)
|
|
301
|
-
|
|
298
|
+
line_definitions.each do |_lt, definition|
|
|
302
299
|
match = definition.matches(line, &warning_handler)
|
|
303
300
|
return match if match
|
|
304
301
|
end
|
|
305
302
|
|
|
306
|
-
|
|
303
|
+
nil
|
|
307
304
|
end
|
|
308
|
-
|
|
305
|
+
|
|
309
306
|
# Returns the max line length for this file format if any.
|
|
310
307
|
def max_line_length
|
|
311
308
|
self.class.const_get(MAX_LINE_LENGTH) if self.class.const_defined?(:MAX_LINE_LENGTH)
|
|
312
309
|
end
|
|
313
|
-
|
|
310
|
+
|
|
314
311
|
def line_divider
|
|
315
312
|
self.class.const_get(LINE_DIVIDER) if self.class.const_defined?(:LINE_DIVIDER)
|
|
316
313
|
end
|