request-log-analyzer 1.3.4 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +3 -3
- data/README.rdoc +1 -1
- data/bin/request-log-analyzer +17 -14
- data/lib/cli/command_line_arguments.rb +51 -51
- data/lib/cli/database_console.rb +3 -3
- data/lib/cli/database_console_init.rb +4 -3
- data/lib/cli/progressbar.rb +10 -10
- data/lib/cli/tools.rb +3 -3
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +19 -17
- data/lib/request_log_analyzer/aggregator/echo.rb +14 -9
- data/lib/request_log_analyzer/aggregator/summarizer.rb +26 -26
- data/lib/request_log_analyzer/aggregator.rb +11 -15
- data/lib/request_log_analyzer/controller.rb +162 -89
- data/lib/request_log_analyzer/database/base.rb +22 -21
- data/lib/request_log_analyzer/database/connection.rb +3 -3
- data/lib/request_log_analyzer/database/request.rb +22 -0
- data/lib/request_log_analyzer/database/source.rb +13 -0
- data/lib/request_log_analyzer/database/warning.rb +14 -0
- data/lib/request_log_analyzer/database.rb +28 -103
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +20 -20
- data/lib/request_log_analyzer/file_format/apache.rb +37 -30
- data/lib/request_log_analyzer/file_format/merb.rb +30 -13
- data/lib/request_log_analyzer/file_format/rack.rb +11 -0
- data/lib/request_log_analyzer/file_format/rails.rb +143 -73
- data/lib/request_log_analyzer/file_format/rails_development.rb +4 -49
- data/lib/request_log_analyzer/file_format.rb +16 -28
- data/lib/request_log_analyzer/filter/anonymize.rb +8 -7
- data/lib/request_log_analyzer/filter/field.rb +9 -9
- data/lib/request_log_analyzer/filter/timespan.rb +12 -10
- data/lib/request_log_analyzer/filter.rb +12 -16
- data/lib/request_log_analyzer/line_definition.rb +15 -14
- data/lib/request_log_analyzer/log_processor.rb +27 -29
- data/lib/request_log_analyzer/mailer.rb +15 -9
- data/lib/request_log_analyzer/output/fixed_width.rb +48 -48
- data/lib/request_log_analyzer/output/html.rb +20 -20
- data/lib/request_log_analyzer/output.rb +53 -12
- data/lib/request_log_analyzer/request.rb +88 -69
- data/lib/request_log_analyzer/source/database_loader.rb +10 -14
- data/lib/request_log_analyzer/source/log_parser.rb +57 -50
- data/lib/request_log_analyzer/source.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +41 -134
- data/lib/request_log_analyzer/tracker/frequency.rb +31 -32
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +21 -21
- data/lib/request_log_analyzer/tracker/timespan.rb +17 -17
- data/lib/request_log_analyzer/tracker/traffic.rb +106 -0
- data/lib/request_log_analyzer/tracker.rb +139 -32
- data/lib/request_log_analyzer.rb +5 -5
- data/request-log-analyzer.gemspec +19 -15
- data/spec/database.yml +6 -0
- data/spec/fixtures/rails_22.log +1 -1
- data/spec/integration/command_line_usage_spec.rb +7 -1
- data/spec/lib/helpers.rb +7 -7
- data/spec/lib/macros.rb +3 -3
- data/spec/lib/matchers.rb +41 -27
- data/spec/lib/mocks.rb +19 -15
- data/spec/lib/testing_format.rb +9 -9
- data/spec/spec_helper.rb +6 -6
- data/spec/unit/aggregator/database_inserter_spec.rb +16 -16
- data/spec/unit/aggregator/summarizer_spec.rb +4 -4
- data/spec/unit/controller/controller_spec.rb +2 -2
- data/spec/unit/controller/log_processor_spec.rb +1 -1
- data/spec/unit/database/base_class_spec.rb +26 -33
- data/spec/unit/database/connection_spec.rb +3 -3
- data/spec/unit/database/database_spec.rb +33 -38
- data/spec/unit/file_format/amazon_s3_format_spec.rb +5 -5
- data/spec/unit/file_format/apache_format_spec.rb +48 -11
- data/spec/unit/file_format/file_format_api_spec.rb +13 -13
- data/spec/unit/file_format/line_definition_spec.rb +24 -17
- data/spec/unit/file_format/merb_format_spec.rb +41 -45
- data/spec/unit/file_format/rails_format_spec.rb +157 -117
- data/spec/unit/filter/anonymize_filter_spec.rb +2 -2
- data/spec/unit/filter/field_filter_spec.rb +13 -13
- data/spec/unit/filter/filter_spec.rb +1 -1
- data/spec/unit/filter/timespan_filter_spec.rb +15 -15
- data/spec/unit/mailer_spec.rb +30 -0
- data/spec/unit/{source/request_spec.rb → request_spec.rb} +30 -30
- data/spec/unit/source/log_parser_spec.rb +27 -27
- data/spec/unit/tracker/duration_tracker_spec.rb +115 -78
- data/spec/unit/tracker/frequency_tracker_spec.rb +74 -63
- data/spec/unit/tracker/hourly_spread_spec.rb +28 -20
- data/spec/unit/tracker/timespan_tracker_spec.rb +25 -13
- data/spec/unit/tracker/tracker_api_spec.rb +117 -42
- data/spec/unit/tracker/traffic_tracker_spec.rb +107 -0
- data/tasks/github-gem.rake +125 -75
- data/tasks/request_log_analyzer.rake +2 -2
- metadata +19 -10
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
2
|
-
|
|
2
|
+
|
|
3
3
|
self.abstract_class = true
|
|
4
4
|
|
|
5
5
|
def <=>(other)
|
|
@@ -14,25 +14,26 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
|
14
14
|
self.class.name.underscore.gsub(/_line$/, '').to_sym
|
|
15
15
|
end
|
|
16
16
|
|
|
17
|
-
|
|
17
|
+
class_inheritable_accessor :line_definition
|
|
18
|
+
cattr_accessor :database
|
|
18
19
|
|
|
19
20
|
def self.subclass_from_line_definition(definition)
|
|
20
21
|
klass = Class.new(RequestLogAnalyzer::Database::Base)
|
|
21
22
|
klass.set_table_name("#{definition.name}_lines")
|
|
22
|
-
|
|
23
|
+
|
|
23
24
|
klass.line_definition = definition
|
|
24
|
-
|
|
25
|
+
|
|
25
26
|
# Set relations with requests and sources table
|
|
26
27
|
klass.belongs_to :request
|
|
27
28
|
klass.belongs_to :source
|
|
28
|
-
|
|
29
|
+
|
|
29
30
|
# Serialize complex fields into the database
|
|
30
31
|
definition.captures.select { |c| c.has_key?(:provides) }.each do |capture|
|
|
31
32
|
klass.send(:serialize, capture[:name], Hash)
|
|
32
33
|
end
|
|
33
34
|
|
|
34
|
-
|
|
35
|
-
|
|
35
|
+
RequestLogAnalyzer::Database::Request.has_many "#{definition.name}_lines".to_sym
|
|
36
|
+
RequestLogAnalyzer::Database::Source.has_many "#{definition.name}_lines".to_sym
|
|
36
37
|
|
|
37
38
|
return klass
|
|
38
39
|
end
|
|
@@ -45,34 +46,34 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
|
45
46
|
|
|
46
47
|
if klass.column_names.include?('request_id')
|
|
47
48
|
klass.belongs_to :request
|
|
48
|
-
|
|
49
|
+
RequestLogAnalyzer::Database::Request.has_many table.to_sym
|
|
49
50
|
end
|
|
50
|
-
|
|
51
|
+
|
|
51
52
|
if klass.column_names.include?('source_id')
|
|
52
53
|
klass.belongs_to :source
|
|
53
|
-
|
|
54
|
+
RequestLogAnalyzer::Database::Source.has_many table.to_sym
|
|
54
55
|
end
|
|
55
|
-
|
|
56
|
+
|
|
56
57
|
return klass
|
|
57
58
|
end
|
|
58
|
-
|
|
59
|
+
|
|
59
60
|
def self.drop_table!
|
|
60
61
|
database.connection.remove_index(self.table_name, [:source_id]) rescue nil
|
|
61
62
|
database.connection.remove_index(self.table_name, [:request_id]) rescue nil
|
|
62
63
|
database.connection.drop_table(self.table_name) if database.connection.table_exists?(self.table_name)
|
|
63
64
|
end
|
|
64
|
-
|
|
65
|
+
|
|
65
66
|
def self.create_table!
|
|
66
67
|
raise "No line_definition available to base table schema on!" unless self.line_definition
|
|
67
|
-
|
|
68
|
+
|
|
68
69
|
unless table_exists?
|
|
69
70
|
database.connection.create_table(table_name.to_sym) do |t|
|
|
70
|
-
|
|
71
|
+
|
|
71
72
|
# Default fields
|
|
72
73
|
t.column :request_id, :integer
|
|
73
74
|
t.column :source_id, :integer
|
|
74
75
|
t.column :lineno, :integer
|
|
75
|
-
|
|
76
|
+
|
|
76
77
|
line_definition.captures.each do |capture|
|
|
77
78
|
# Add a field for every capture
|
|
78
79
|
t.column(capture[:name], column_type(capture[:type]))
|
|
@@ -82,13 +83,13 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
|
82
83
|
end
|
|
83
84
|
end
|
|
84
85
|
end
|
|
85
|
-
|
|
86
|
+
|
|
86
87
|
# Add indices to table for more speedy querying
|
|
87
88
|
database.connection.add_index(self.table_name.to_sym, [:request_id]) # rescue
|
|
88
89
|
database.connection.add_index(self.table_name.to_sym, [:source_id]) # rescue
|
|
89
90
|
end
|
|
90
|
-
|
|
91
|
-
|
|
91
|
+
|
|
92
|
+
|
|
92
93
|
# Function to determine the column type for a field
|
|
93
94
|
# TODO: make more robust / include in file-format definition
|
|
94
95
|
def self.column_type(type_indicator)
|
|
@@ -109,6 +110,6 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
|
109
110
|
when :date; :date
|
|
110
111
|
else :string
|
|
111
112
|
end
|
|
112
|
-
end
|
|
113
|
-
|
|
113
|
+
end
|
|
114
|
+
|
|
114
115
|
end
|
|
@@ -26,13 +26,13 @@ module RequestLogAnalyzer::Database::Connection
|
|
|
26
26
|
raise "Cannot connect with this connection_identifier: #{connection_identifier.inspect}"
|
|
27
27
|
end
|
|
28
28
|
end
|
|
29
|
-
|
|
29
|
+
|
|
30
30
|
def disconnect
|
|
31
31
|
RequestLogAnalyzer::Database::Base.remove_connection
|
|
32
32
|
end
|
|
33
|
-
|
|
33
|
+
|
|
34
34
|
def connection
|
|
35
35
|
RequestLogAnalyzer::Database::Base.connection
|
|
36
36
|
end
|
|
37
|
-
|
|
37
|
+
|
|
38
38
|
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base
|
|
2
|
+
|
|
3
|
+
# Returns an array of all the Line objects of this request in the correct order.
|
|
4
|
+
def lines
|
|
5
|
+
@lines ||= begin
|
|
6
|
+
lines = []
|
|
7
|
+
self.class.reflections.each { |r, d| lines += self.send(r).all }
|
|
8
|
+
lines.sort
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
# Creates the table to store requests in.
|
|
13
|
+
def self.create_table!
|
|
14
|
+
unless database.connection.table_exists?(:requests)
|
|
15
|
+
database.connection.create_table(:requests) do |t|
|
|
16
|
+
t.column :first_lineno, :integer
|
|
17
|
+
t.column :last_lineno, :integer
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
class RequestLogAnalyzer::Database::Source < RequestLogAnalyzer::Database::Base
|
|
2
|
+
|
|
3
|
+
def self.create_table!
|
|
4
|
+
unless database.connection.table_exists?(:sources)
|
|
5
|
+
database.connection.create_table(:sources) do |t|
|
|
6
|
+
t.column :filename, :string
|
|
7
|
+
t.column :mtime, :datetime
|
|
8
|
+
t.column :filesize, :integer
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
class RequestLogAnalyzer::Database::Warning < RequestLogAnalyzer::Database::Base
|
|
2
|
+
|
|
3
|
+
def self.create_table!
|
|
4
|
+
unless database.connection.table_exists?(:warnings)
|
|
5
|
+
database.connection.create_table(:warnings) do |t|
|
|
6
|
+
t.column :warning_type, :string, :limit => 30, :null => false
|
|
7
|
+
t.column :message, :string
|
|
8
|
+
t.column :source_id, :integer
|
|
9
|
+
t.column :lineno, :integer
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
end
|
|
@@ -5,13 +5,13 @@ class RequestLogAnalyzer::Database
|
|
|
5
5
|
|
|
6
6
|
def self.const_missing(const) # :nodoc:
|
|
7
7
|
RequestLogAnalyzer::load_default_class_file(self, const)
|
|
8
|
-
end
|
|
8
|
+
end
|
|
9
9
|
|
|
10
10
|
include RequestLogAnalyzer::Database::Connection
|
|
11
11
|
|
|
12
12
|
attr_accessor :file_format
|
|
13
|
-
attr_reader :
|
|
14
|
-
|
|
13
|
+
attr_reader :line_classes
|
|
14
|
+
|
|
15
15
|
def initialize(connection_identifier = nil)
|
|
16
16
|
@line_classes = []
|
|
17
17
|
RequestLogAnalyzer::Database::Base.database = self
|
|
@@ -23,112 +23,32 @@ class RequestLogAnalyzer::Database
|
|
|
23
23
|
line_type = line_type.name if line_type.respond_to?(:name)
|
|
24
24
|
Object.const_get("#{line_type}_line".camelize)
|
|
25
25
|
end
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
# It will create the class if not previously done so. The class will
|
|
30
|
-
# include a create_table! method the migrate the database.
|
|
31
|
-
def request_class
|
|
32
|
-
@request_class ||= begin
|
|
33
|
-
klass = Class.new(RequestLogAnalyzer::Database::Base) do
|
|
34
|
-
|
|
35
|
-
def lines
|
|
36
|
-
@lines ||= begin
|
|
37
|
-
lines = []
|
|
38
|
-
self.class.reflections.each { |r, d| lines += self.send(r).all }
|
|
39
|
-
lines.sort
|
|
40
|
-
end
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
# Creates the requests table
|
|
44
|
-
def self.create_table!
|
|
45
|
-
unless database.connection.table_exists?(:requests)
|
|
46
|
-
database.connection.create_table(:requests) do |t|
|
|
47
|
-
t.column :first_lineno, :integer
|
|
48
|
-
t.column :last_lineno, :integer
|
|
49
|
-
end
|
|
50
|
-
end
|
|
51
|
-
end
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
Object.const_set('Request', klass)
|
|
55
|
-
Object.const_get('Request')
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
# Returns the Source ORM class for the current database.
|
|
60
|
-
#
|
|
61
|
-
# It will create the class if not previously done so. The class will
|
|
62
|
-
# include a create_table! method the migrate the database.
|
|
63
|
-
def source_class
|
|
64
|
-
@source_class ||= begin
|
|
65
|
-
klass = Class.new(RequestLogAnalyzer::Database::Base) do
|
|
66
|
-
|
|
67
|
-
# Creates the sources table
|
|
68
|
-
def self.create_table!
|
|
69
|
-
unless database.connection.table_exists?(:sources)
|
|
70
|
-
database.connection.create_table(:sources) do |t|
|
|
71
|
-
t.column :filename, :string
|
|
72
|
-
t.column :mtime, :datetime
|
|
73
|
-
t.column :filesize, :integer
|
|
74
|
-
end
|
|
75
|
-
end
|
|
76
|
-
end
|
|
77
|
-
end
|
|
78
|
-
|
|
79
|
-
Object.const_set('Source', klass)
|
|
80
|
-
Object.const_get('Source')
|
|
81
|
-
end
|
|
82
|
-
end
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
# Returns the Warning ORM class for the current database.
|
|
86
|
-
#
|
|
87
|
-
# It will create the class if not previously done so. The class will
|
|
88
|
-
# include a create_table! method the migrate the database.
|
|
89
|
-
def warning_class
|
|
90
|
-
@warning_class ||= begin
|
|
91
|
-
klass = Class.new(RequestLogAnalyzer::Database::Base) do
|
|
92
|
-
|
|
93
|
-
# Creates the warnings table
|
|
94
|
-
def self.create_table!
|
|
95
|
-
unless database.connection.table_exists?(:warnings)
|
|
96
|
-
database.connection.create_table(:warnings) do |t|
|
|
97
|
-
t.column :warning_type, :string, :limit => 30, :null => false
|
|
98
|
-
t.column :message, :string
|
|
99
|
-
t.column :source_id, :integer
|
|
100
|
-
t.column :lineno, :integer
|
|
101
|
-
end
|
|
102
|
-
end
|
|
103
|
-
end
|
|
104
|
-
end
|
|
105
|
-
|
|
106
|
-
Object.const_set('Warning', klass)
|
|
107
|
-
Object.const_get('Warning')
|
|
108
|
-
end
|
|
26
|
+
|
|
27
|
+
def default_classes
|
|
28
|
+
[RequestLogAnalyzer::Database::Request, RequestLogAnalyzer::Database::Source, RequestLogAnalyzer::Database::Warning]
|
|
109
29
|
end
|
|
110
|
-
|
|
30
|
+
|
|
111
31
|
# Loads the ORM classes by inspecting the tables in the current database
|
|
112
32
|
def load_database_schema!
|
|
113
33
|
connection.tables.map do |table|
|
|
114
34
|
case table.to_sym
|
|
115
|
-
when :warnings then
|
|
116
|
-
when :sources then
|
|
117
|
-
when :requests then
|
|
35
|
+
when :warnings then RequestLogAnalyzer::Database::Warning
|
|
36
|
+
when :sources then RequestLogAnalyzer::Database::Source
|
|
37
|
+
when :requests then RequestLogAnalyzer::Database::Request
|
|
118
38
|
else load_activerecord_class(table)
|
|
119
39
|
end
|
|
120
40
|
end
|
|
121
41
|
end
|
|
122
|
-
|
|
42
|
+
|
|
123
43
|
# Returns an array of all the ActiveRecord-bases ORM classes for this database
|
|
124
44
|
def orm_classes
|
|
125
|
-
|
|
45
|
+
default_classes + line_classes
|
|
126
46
|
end
|
|
127
|
-
|
|
47
|
+
|
|
128
48
|
# Loads an ActiveRecord-based class that correspond to the given parameter, which can either be
|
|
129
49
|
# a table name or a LineDefinition instance.
|
|
130
50
|
def load_activerecord_class(linedefinition_or_table)
|
|
131
|
-
|
|
51
|
+
|
|
132
52
|
case linedefinition_or_table
|
|
133
53
|
when String, Symbol
|
|
134
54
|
klass_name = linedefinition_or_table.to_s.singularize.camelize
|
|
@@ -137,39 +57,44 @@ class RequestLogAnalyzer::Database
|
|
|
137
57
|
klass_name = "#{linedefinition_or_table.name}_line".camelize
|
|
138
58
|
klass = RequestLogAnalyzer::Database::Base.subclass_from_line_definition(linedefinition_or_table)
|
|
139
59
|
end
|
|
140
|
-
|
|
60
|
+
|
|
141
61
|
Object.const_set(klass_name, klass)
|
|
142
62
|
klass = Object.const_get(klass_name)
|
|
143
63
|
@line_classes << klass
|
|
144
64
|
return klass
|
|
145
|
-
end
|
|
65
|
+
end
|
|
146
66
|
|
|
147
67
|
def fileformat_classes
|
|
148
68
|
raise "No file_format provided!" unless file_format
|
|
149
|
-
|
|
150
|
-
default_classes = [request_class, source_class, warning_class]
|
|
151
69
|
line_classes = file_format.line_definitions.map { |(name, definition)| load_activerecord_class(definition) }
|
|
152
70
|
return default_classes + line_classes
|
|
153
71
|
end
|
|
154
72
|
|
|
155
|
-
# Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
|
|
73
|
+
# Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
|
|
156
74
|
# file format definition. These ORM classes will later be used to create records in the database.
|
|
157
75
|
def create_database_schema!
|
|
158
76
|
fileformat_classes.each { |klass| klass.create_table! }
|
|
159
77
|
end
|
|
160
|
-
|
|
78
|
+
|
|
161
79
|
# Drops the table of all the ORM classes, and unregisters the classes
|
|
162
80
|
def drop_database_schema!
|
|
163
81
|
file_format ? fileformat_classes.map(&:drop_table!) : orm_classes.map(&:drop_table!)
|
|
164
82
|
remove_orm_classes!
|
|
165
83
|
end
|
|
166
|
-
|
|
84
|
+
|
|
85
|
+
# Registers the default ORM classes in the default namespace
|
|
86
|
+
def register_default_orm_classes!
|
|
87
|
+
Object.const_set('Request', RequestLogAnalyzer::Database::Request)
|
|
88
|
+
Object.const_set('Source', RequestLogAnalyzer::Database::Source)
|
|
89
|
+
Object.const_set('Warning', RequestLogAnalyzer::Database::Warning)
|
|
90
|
+
end
|
|
91
|
+
|
|
167
92
|
# Unregisters every ORM class constant
|
|
168
93
|
def remove_orm_classes!
|
|
169
94
|
orm_classes.each do |klass|
|
|
170
95
|
if klass.respond_to?(:name) && !klass.name.blank?
|
|
171
|
-
|
|
172
|
-
Object.send(:remove_const,
|
|
96
|
+
klass_name = klass.name.split('::').last
|
|
97
|
+
Object.send(:remove_const, klass_name) if Object.const_defined?(klass_name)
|
|
173
98
|
end
|
|
174
99
|
end
|
|
175
100
|
end
|
|
@@ -1,71 +1,71 @@
|
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
|
2
2
|
|
|
3
|
-
# FileFormat for Amazon S3 access logs.
|
|
3
|
+
# FileFormat for Amazon S3 access logs.
|
|
4
4
|
#
|
|
5
5
|
# Access logs are disabled by default on Amazon S3. To enable logging, see
|
|
6
6
|
# http://docs.amazonwebservices.com/AmazonS3/latest/index.html?ServerLogs.html
|
|
7
7
|
class AmazonS3 < Base
|
|
8
|
-
|
|
8
|
+
|
|
9
9
|
line_definition :access do |line|
|
|
10
10
|
line.header = true
|
|
11
11
|
line.footer = true
|
|
12
|
-
line.regexp = /^([^\ ]+) ([^\ ]+) \[([
|
|
13
|
-
line.captures << { :name => :bucket_owner, :type => :string } <<
|
|
14
|
-
{ :name => :bucket, :type => :string } <<
|
|
12
|
+
line.regexp = /^([^\ ]+) ([^\ ]+) \[(\d{2}\/[A-Za-z]{3}\/\d{4}.\d{2}:\d{2}:\d{2})(?: .\d{4})?\] (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]+)" "([^"]+)"/
|
|
13
|
+
line.captures << { :name => :bucket_owner, :type => :string } <<
|
|
14
|
+
{ :name => :bucket, :type => :string } <<
|
|
15
15
|
{ :name => :timestamp, :type => :timestamp } <<
|
|
16
16
|
{ :name => :remote_ip, :type => :string } <<
|
|
17
|
-
{ :name => :requester, :type => :string } <<
|
|
17
|
+
{ :name => :requester, :type => :string } <<
|
|
18
18
|
{ :name => :request_id, :type => :string } <<
|
|
19
19
|
{ :name => :operation, :type => :string } <<
|
|
20
20
|
{ :name => :key, :type => :nillable_string } <<
|
|
21
21
|
{ :name => :request_uri, :type => :string } <<
|
|
22
22
|
{ :name => :http_status, :type => :integer } <<
|
|
23
23
|
{ :name => :error_code, :type => :nillable_string } <<
|
|
24
|
-
{ :name => :bytes_sent, :type => :
|
|
25
|
-
{ :name => :object_size, :type => :
|
|
24
|
+
{ :name => :bytes_sent, :type => :traffic, :unit => :byte } <<
|
|
25
|
+
{ :name => :object_size, :type => :traffic, :unit => :byte } <<
|
|
26
26
|
{ :name => :total_time, :type => :duration, :unit => :msec } <<
|
|
27
27
|
{ :name => :turnaround_time, :type => :duration, :unit => :msec } <<
|
|
28
28
|
{ :name => :referer, :type => :referer } <<
|
|
29
29
|
{ :name => :user_agent, :type => :user_agent }
|
|
30
30
|
end
|
|
31
|
-
|
|
31
|
+
|
|
32
32
|
report do |analyze|
|
|
33
33
|
analyze.timespan
|
|
34
34
|
analyze.hourly_spread
|
|
35
35
|
|
|
36
|
-
analyze.frequency :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :
|
|
37
|
-
analyze.duration :duration => :total_time, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :
|
|
36
|
+
analyze.frequency :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Most popular items"
|
|
37
|
+
analyze.duration :duration => :total_time, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Request duration"
|
|
38
|
+
analyze.traffic :traffic => :bytes_sent, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Traffic"
|
|
38
39
|
analyze.frequency :category => :http_status, :title => 'HTTP status codes'
|
|
39
40
|
analyze.frequency :category => :error_code, :title => 'Error codes'
|
|
40
41
|
end
|
|
41
|
-
|
|
42
|
+
|
|
42
43
|
class Request < RequestLogAnalyzer::Request
|
|
43
|
-
|
|
44
|
+
|
|
44
45
|
MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
|
|
45
46
|
'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
|
|
46
|
-
|
|
47
|
+
|
|
47
48
|
# Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
|
|
48
49
|
# to speed up parsing.
|
|
49
50
|
def convert_timestamp(value, definition)
|
|
50
|
-
|
|
51
|
-
"#{d[2]}#{MONTHS[d[1]]}#{d[0]}#{d[3]}#{d[4]}#{d[5]}".to_i
|
|
51
|
+
"#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
|
|
52
52
|
end
|
|
53
|
-
|
|
53
|
+
|
|
54
54
|
# Make sure that the string '-' is parsed as a nil value.
|
|
55
55
|
def convert_nillable_string(value, definition)
|
|
56
56
|
value == '-' ? nil : value
|
|
57
57
|
end
|
|
58
|
-
|
|
58
|
+
|
|
59
59
|
# Can be implemented in subclasses for improved categorizations
|
|
60
60
|
def convert_referer(value, definition)
|
|
61
61
|
value == '-' ? nil : value
|
|
62
62
|
end
|
|
63
|
-
|
|
63
|
+
|
|
64
64
|
# Can be implemented in subclasses for improved categorizations
|
|
65
65
|
def convert_user_agent(value, definition)
|
|
66
66
|
value == '-' ? nil : value
|
|
67
67
|
end
|
|
68
68
|
end
|
|
69
|
-
|
|
69
|
+
|
|
70
70
|
end
|
|
71
71
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
|
2
|
-
|
|
3
|
-
# The Apache file format is able to log Apache access.log files.
|
|
2
|
+
|
|
3
|
+
# The Apache file format is able to log Apache access.log files.
|
|
4
4
|
#
|
|
5
5
|
# The access.log can be configured in Apache to have many different formats. In theory, this
|
|
6
6
|
# FileFormat can handle any format, but it must be aware of the log formatting that is used
|
|
@@ -17,7 +17,10 @@ module RequestLogAnalyzer::FileFormat
|
|
|
17
17
|
# A hash of predefined Apache log formats
|
|
18
18
|
LOG_FORMAT_DEFAULTS = {
|
|
19
19
|
:common => '%h %l %u %t "%r" %>s %b',
|
|
20
|
-
:combined => '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"'
|
|
20
|
+
:combined => '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"',
|
|
21
|
+
:rack => '%h %l %u %t "%r" %>s %b %T',
|
|
22
|
+
:referer => '%{Referer}i -> %U',
|
|
23
|
+
:agent => '%{User-agent}i'
|
|
21
24
|
}
|
|
22
25
|
|
|
23
26
|
# A hash that defines how the log format directives should be parsed.
|
|
@@ -25,18 +28,19 @@ module RequestLogAnalyzer::FileFormat
|
|
|
25
28
|
'%' => { :regexp => '%', :captures => [] },
|
|
26
29
|
'h' => { :regexp => '([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+)', :captures => [{:name => :remote_host, :type => :string}] },
|
|
27
30
|
'a' => { :regexp => '(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', :captures => [{:name => :remote_ip, :type => :string}] },
|
|
28
|
-
'b' => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :
|
|
31
|
+
'b' => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] },
|
|
29
32
|
'c' => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] },
|
|
30
|
-
'D' => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :
|
|
33
|
+
'D' => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :musec}] },
|
|
31
34
|
'l' => { :regexp => '([\w-]+)', :captures => [{:name => :remote_logname, :type => :nillable_string}] },
|
|
32
|
-
'T' => { :regexp => '(
|
|
33
|
-
't' => { :regexp => '\[([
|
|
35
|
+
'T' => { :regexp => '((?:\d+(?:\.\d+))|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] },
|
|
36
|
+
't' => { :regexp => '\[(\d{2}\/[A-Za-z]{3}\/\d{4}.\d{2}:\d{2}:\d{2})(?: .\d{4})?\]', :captures => [{:name => :timestamp, :type => :timestamp}] },
|
|
34
37
|
's' => { :regexp => '(\d{3})', :captures => [{:name => :http_status, :type => :integer}] },
|
|
35
38
|
'u' => { :regexp => '(\w+|-)', :captures => [{:name => :user, :type => :nillable_string}] },
|
|
36
|
-
'
|
|
39
|
+
'U' => { :regexp => '(\/\S*)', :captures => [{:name => :path, :type => :string}] },
|
|
40
|
+
'r' => { :regexp => '([A-Z]+) (\S+) HTTP\/(\d+(?:\.\d+)*)', :captures => [{:name => :http_method, :type => :string},
|
|
37
41
|
{:name => :path, :type => :path}, {:name => :http_version, :type => :string}]},
|
|
38
|
-
'i' => { 'Referer' => { :regexp => '(
|
|
39
|
-
'User-agent' => { :regexp => '(.*)',
|
|
42
|
+
'i' => { 'Referer' => { :regexp => '(\S+)', :captures => [{:name => :referer, :type => :nillable_string}] },
|
|
43
|
+
'User-agent' => { :regexp => '(.*)', :captures => [{:name => :user_agent, :type => :user_agent}] }
|
|
40
44
|
}
|
|
41
45
|
}
|
|
42
46
|
|
|
@@ -44,8 +48,9 @@ module RequestLogAnalyzer::FileFormat
|
|
|
44
48
|
# It will set up the line definition and the report trackers according to the Apache access log format,
|
|
45
49
|
# which should be passed as first argument. By default, is uses the 'combined' log format.
|
|
46
50
|
def self.create(*args)
|
|
47
|
-
access_line =
|
|
48
|
-
|
|
51
|
+
access_line = access_line_definition(args.first)
|
|
52
|
+
trackers = report_trackers(access_line) + report_definer.trackers
|
|
53
|
+
self.new(line_definer.line_definitions.merge(:access => access_line), trackers)
|
|
49
54
|
end
|
|
50
55
|
|
|
51
56
|
# Creates the access log line definition based on the Apache log format string
|
|
@@ -58,7 +63,7 @@ module RequestLogAnalyzer::FileFormat
|
|
|
58
63
|
format_string.scan(/([^%]*)(?:%(?:\{([^\}]+)\})?>?([A-Za-z%]))?/) do |literal, arg, variable|
|
|
59
64
|
|
|
60
65
|
line_regexp << Regexp.quote(literal) # Make sure to parse the literal before the directive
|
|
61
|
-
|
|
66
|
+
|
|
62
67
|
if variable
|
|
63
68
|
# Check if we recognize the log directive
|
|
64
69
|
directive = LOG_DIRECTIVES[variable]
|
|
@@ -73,7 +78,7 @@ module RequestLogAnalyzer::FileFormat
|
|
|
73
78
|
end
|
|
74
79
|
end
|
|
75
80
|
end
|
|
76
|
-
|
|
81
|
+
|
|
77
82
|
# Return a new line definition object
|
|
78
83
|
return RequestLogAnalyzer::LineDefinition.new(:access, :regexp => Regexp.new(line_regexp),
|
|
79
84
|
:captures => captures, :header => true, :footer => true)
|
|
@@ -86,45 +91,47 @@ module RequestLogAnalyzer::FileFormat
|
|
|
86
91
|
analyze.timespan if line_definition.captures?(:timestamp)
|
|
87
92
|
analyze.hourly_spread if line_definition.captures?(:timestamp)
|
|
88
93
|
|
|
89
|
-
analyze.frequency :category => :http_method, :
|
|
90
|
-
analyze.frequency :category => :http_status, :
|
|
91
|
-
analyze.frequency :category =>
|
|
94
|
+
analyze.frequency :category => :http_method, :title => "HTTP methods" if line_definition.captures?(:http_method)
|
|
95
|
+
analyze.frequency :category => :http_status, :title => "HTTP statuses" if line_definition.captures?(:http_status)
|
|
96
|
+
analyze.frequency :category => lambda { |r| r.category }, :title => "Most popular URIs" if line_definition.captures?(:path)
|
|
92
97
|
|
|
93
|
-
analyze.frequency :category => :user_agent, :
|
|
94
|
-
analyze.frequency :category => :referer, :
|
|
98
|
+
analyze.frequency :category => :user_agent, :title => "User agents" if line_definition.captures?(:user_agent)
|
|
99
|
+
analyze.frequency :category => :referer, :title => "Referers" if line_definition.captures?(:referer)
|
|
95
100
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
end
|
|
101
|
+
analyze.duration :duration => :duration, :category => lambda { |r| r.category }, :title => 'Request duration' if line_definition.captures?(:duration)
|
|
102
|
+
analyze.traffic :traffic => :bytes_sent, :category => lambda { |r| r.category }, :title => 'Traffic' if line_definition.captures?(:bytes_sent)
|
|
99
103
|
|
|
100
104
|
return analyze.trackers
|
|
101
105
|
end
|
|
102
106
|
|
|
103
107
|
# Define a custom Request class for the Apache file format to speed up timestamp handling.
|
|
104
108
|
class Request < RequestLogAnalyzer::Request
|
|
105
|
-
|
|
109
|
+
|
|
110
|
+
def category
|
|
111
|
+
first(:path)
|
|
112
|
+
end
|
|
113
|
+
|
|
106
114
|
MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
|
|
107
115
|
'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
|
|
108
|
-
|
|
116
|
+
|
|
109
117
|
# Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
|
|
110
118
|
# to speed up parsing.
|
|
111
119
|
def convert_timestamp(value, definition)
|
|
112
|
-
|
|
113
|
-
"#{d[2]}#{MONTHS[d[1]]}#{d[0]}#{d[3]}#{d[4]}#{d[5]}".to_i
|
|
120
|
+
"#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
|
|
114
121
|
end
|
|
115
|
-
|
|
122
|
+
|
|
116
123
|
# This function can be overridden to rewrite the path for better categorization in the
|
|
117
124
|
# reports.
|
|
118
125
|
def convert_path(value, definition)
|
|
119
126
|
value
|
|
120
127
|
end
|
|
121
|
-
|
|
122
|
-
# This function can be overridden to simplify the user agent string for better
|
|
128
|
+
|
|
129
|
+
# This function can be overridden to simplify the user agent string for better
|
|
123
130
|
# categorization in the reports
|
|
124
131
|
def convert_user_agent(value, definition)
|
|
125
132
|
value # TODO
|
|
126
133
|
end
|
|
127
|
-
|
|
134
|
+
|
|
128
135
|
# Make sure that the string '-' is parsed as a nil value.
|
|
129
136
|
def convert_nillable_string(value, definition)
|
|
130
137
|
value == '-' ? nil : value
|