request-log-analyzer 1.3.5 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +3 -3
- data/README.rdoc +1 -1
- data/bin/request-log-analyzer +17 -14
- data/lib/cli/command_line_arguments.rb +51 -51
- data/lib/cli/database_console.rb +3 -3
- data/lib/cli/database_console_init.rb +4 -3
- data/lib/cli/progressbar.rb +10 -10
- data/lib/cli/tools.rb +3 -3
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +13 -14
- data/lib/request_log_analyzer/aggregator/echo.rb +14 -9
- data/lib/request_log_analyzer/aggregator/summarizer.rb +26 -26
- data/lib/request_log_analyzer/aggregator.rb +11 -15
- data/lib/request_log_analyzer/controller.rb +162 -89
- data/lib/request_log_analyzer/database/base.rb +21 -21
- data/lib/request_log_analyzer/database/connection.rb +3 -3
- data/lib/request_log_analyzer/database/request.rb +22 -0
- data/lib/request_log_analyzer/database/source.rb +13 -0
- data/lib/request_log_analyzer/database/warning.rb +14 -0
- data/lib/request_log_analyzer/database.rb +28 -103
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +17 -18
- data/lib/request_log_analyzer/file_format/apache.rb +26 -27
- data/lib/request_log_analyzer/file_format/merb.rb +30 -13
- data/lib/request_log_analyzer/file_format/rack.rb +4 -4
- data/lib/request_log_analyzer/file_format/rails.rb +143 -73
- data/lib/request_log_analyzer/file_format/rails_development.rb +4 -49
- data/lib/request_log_analyzer/file_format.rb +16 -28
- data/lib/request_log_analyzer/filter/anonymize.rb +8 -7
- data/lib/request_log_analyzer/filter/field.rb +9 -9
- data/lib/request_log_analyzer/filter/timespan.rb +12 -10
- data/lib/request_log_analyzer/filter.rb +12 -16
- data/lib/request_log_analyzer/line_definition.rb +15 -14
- data/lib/request_log_analyzer/log_processor.rb +27 -29
- data/lib/request_log_analyzer/mailer.rb +15 -9
- data/lib/request_log_analyzer/output/fixed_width.rb +40 -41
- data/lib/request_log_analyzer/output/html.rb +20 -20
- data/lib/request_log_analyzer/output.rb +53 -12
- data/lib/request_log_analyzer/request.rb +75 -68
- data/lib/request_log_analyzer/source/database_loader.rb +10 -14
- data/lib/request_log_analyzer/source/log_parser.rb +57 -50
- data/lib/request_log_analyzer/source.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +39 -132
- data/lib/request_log_analyzer/tracker/frequency.rb +31 -32
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +21 -21
- data/lib/request_log_analyzer/tracker/timespan.rb +17 -17
- data/lib/request_log_analyzer/tracker/traffic.rb +36 -116
- data/lib/request_log_analyzer/tracker.rb +139 -32
- data/lib/request_log_analyzer.rb +4 -4
- data/request-log-analyzer.gemspec +19 -15
- data/spec/database.yml +6 -0
- data/spec/fixtures/rails_22.log +1 -1
- data/spec/integration/command_line_usage_spec.rb +7 -1
- data/spec/lib/helpers.rb +7 -7
- data/spec/lib/macros.rb +3 -3
- data/spec/lib/matchers.rb +41 -27
- data/spec/lib/mocks.rb +15 -14
- data/spec/lib/testing_format.rb +9 -9
- data/spec/spec_helper.rb +6 -6
- data/spec/unit/aggregator/database_inserter_spec.rb +16 -16
- data/spec/unit/aggregator/summarizer_spec.rb +4 -4
- data/spec/unit/controller/controller_spec.rb +2 -2
- data/spec/unit/controller/log_processor_spec.rb +1 -1
- data/spec/unit/database/base_class_spec.rb +26 -33
- data/spec/unit/database/connection_spec.rb +3 -3
- data/spec/unit/database/database_spec.rb +33 -38
- data/spec/unit/file_format/amazon_s3_format_spec.rb +5 -5
- data/spec/unit/file_format/apache_format_spec.rb +13 -13
- data/spec/unit/file_format/file_format_api_spec.rb +13 -13
- data/spec/unit/file_format/line_definition_spec.rb +24 -17
- data/spec/unit/file_format/merb_format_spec.rb +41 -45
- data/spec/unit/file_format/rails_format_spec.rb +157 -117
- data/spec/unit/filter/anonymize_filter_spec.rb +2 -2
- data/spec/unit/filter/field_filter_spec.rb +13 -13
- data/spec/unit/filter/filter_spec.rb +1 -1
- data/spec/unit/filter/timespan_filter_spec.rb +15 -15
- data/spec/unit/mailer_spec.rb +30 -0
- data/spec/unit/{source/request_spec.rb → request_spec.rb} +30 -30
- data/spec/unit/source/log_parser_spec.rb +27 -27
- data/spec/unit/tracker/duration_tracker_spec.rb +115 -78
- data/spec/unit/tracker/frequency_tracker_spec.rb +74 -63
- data/spec/unit/tracker/hourly_spread_spec.rb +28 -20
- data/spec/unit/tracker/timespan_tracker_spec.rb +25 -13
- data/spec/unit/tracker/tracker_api_spec.rb +117 -42
- data/spec/unit/tracker/traffic_tracker_spec.rb +81 -79
- data/tasks/github-gem.rake +125 -75
- data/tasks/request_log_analyzer.rake +2 -2
- metadata +13 -8
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
2
|
-
|
|
2
|
+
|
|
3
3
|
self.abstract_class = true
|
|
4
4
|
|
|
5
5
|
def <=>(other)
|
|
@@ -13,27 +13,27 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
|
13
13
|
def line_type
|
|
14
14
|
self.class.name.underscore.gsub(/_line$/, '').to_sym
|
|
15
15
|
end
|
|
16
|
-
|
|
16
|
+
|
|
17
17
|
class_inheritable_accessor :line_definition
|
|
18
18
|
cattr_accessor :database
|
|
19
19
|
|
|
20
20
|
def self.subclass_from_line_definition(definition)
|
|
21
21
|
klass = Class.new(RequestLogAnalyzer::Database::Base)
|
|
22
22
|
klass.set_table_name("#{definition.name}_lines")
|
|
23
|
-
|
|
23
|
+
|
|
24
24
|
klass.line_definition = definition
|
|
25
|
-
|
|
25
|
+
|
|
26
26
|
# Set relations with requests and sources table
|
|
27
27
|
klass.belongs_to :request
|
|
28
28
|
klass.belongs_to :source
|
|
29
|
-
|
|
29
|
+
|
|
30
30
|
# Serialize complex fields into the database
|
|
31
31
|
definition.captures.select { |c| c.has_key?(:provides) }.each do |capture|
|
|
32
32
|
klass.send(:serialize, capture[:name], Hash)
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
-
|
|
36
|
-
|
|
35
|
+
RequestLogAnalyzer::Database::Request.has_many "#{definition.name}_lines".to_sym
|
|
36
|
+
RequestLogAnalyzer::Database::Source.has_many "#{definition.name}_lines".to_sym
|
|
37
37
|
|
|
38
38
|
return klass
|
|
39
39
|
end
|
|
@@ -46,34 +46,34 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
|
46
46
|
|
|
47
47
|
if klass.column_names.include?('request_id')
|
|
48
48
|
klass.belongs_to :request
|
|
49
|
-
|
|
49
|
+
RequestLogAnalyzer::Database::Request.has_many table.to_sym
|
|
50
50
|
end
|
|
51
|
-
|
|
51
|
+
|
|
52
52
|
if klass.column_names.include?('source_id')
|
|
53
53
|
klass.belongs_to :source
|
|
54
|
-
|
|
54
|
+
RequestLogAnalyzer::Database::Source.has_many table.to_sym
|
|
55
55
|
end
|
|
56
|
-
|
|
56
|
+
|
|
57
57
|
return klass
|
|
58
58
|
end
|
|
59
|
-
|
|
59
|
+
|
|
60
60
|
def self.drop_table!
|
|
61
61
|
database.connection.remove_index(self.table_name, [:source_id]) rescue nil
|
|
62
62
|
database.connection.remove_index(self.table_name, [:request_id]) rescue nil
|
|
63
63
|
database.connection.drop_table(self.table_name) if database.connection.table_exists?(self.table_name)
|
|
64
64
|
end
|
|
65
|
-
|
|
65
|
+
|
|
66
66
|
def self.create_table!
|
|
67
67
|
raise "No line_definition available to base table schema on!" unless self.line_definition
|
|
68
|
-
|
|
68
|
+
|
|
69
69
|
unless table_exists?
|
|
70
70
|
database.connection.create_table(table_name.to_sym) do |t|
|
|
71
|
-
|
|
71
|
+
|
|
72
72
|
# Default fields
|
|
73
73
|
t.column :request_id, :integer
|
|
74
74
|
t.column :source_id, :integer
|
|
75
75
|
t.column :lineno, :integer
|
|
76
|
-
|
|
76
|
+
|
|
77
77
|
line_definition.captures.each do |capture|
|
|
78
78
|
# Add a field for every capture
|
|
79
79
|
t.column(capture[:name], column_type(capture[:type]))
|
|
@@ -83,13 +83,13 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
|
83
83
|
end
|
|
84
84
|
end
|
|
85
85
|
end
|
|
86
|
-
|
|
86
|
+
|
|
87
87
|
# Add indices to table for more speedy querying
|
|
88
88
|
database.connection.add_index(self.table_name.to_sym, [:request_id]) # rescue
|
|
89
89
|
database.connection.add_index(self.table_name.to_sym, [:source_id]) # rescue
|
|
90
90
|
end
|
|
91
|
-
|
|
92
|
-
|
|
91
|
+
|
|
92
|
+
|
|
93
93
|
# Function to determine the column type for a field
|
|
94
94
|
# TODO: make more robust / include in file-format definition
|
|
95
95
|
def self.column_type(type_indicator)
|
|
@@ -110,6 +110,6 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
|
110
110
|
when :date; :date
|
|
111
111
|
else :string
|
|
112
112
|
end
|
|
113
|
-
end
|
|
114
|
-
|
|
113
|
+
end
|
|
114
|
+
|
|
115
115
|
end
|
|
@@ -26,13 +26,13 @@ module RequestLogAnalyzer::Database::Connection
|
|
|
26
26
|
raise "Cannot connect with this connection_identifier: #{connection_identifier.inspect}"
|
|
27
27
|
end
|
|
28
28
|
end
|
|
29
|
-
|
|
29
|
+
|
|
30
30
|
def disconnect
|
|
31
31
|
RequestLogAnalyzer::Database::Base.remove_connection
|
|
32
32
|
end
|
|
33
|
-
|
|
33
|
+
|
|
34
34
|
def connection
|
|
35
35
|
RequestLogAnalyzer::Database::Base.connection
|
|
36
36
|
end
|
|
37
|
-
|
|
37
|
+
|
|
38
38
|
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base
|
|
2
|
+
|
|
3
|
+
# Returns an array of all the Line objects of this request in the correct order.
|
|
4
|
+
def lines
|
|
5
|
+
@lines ||= begin
|
|
6
|
+
lines = []
|
|
7
|
+
self.class.reflections.each { |r, d| lines += self.send(r).all }
|
|
8
|
+
lines.sort
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
# Creates the table to store requests in.
|
|
13
|
+
def self.create_table!
|
|
14
|
+
unless database.connection.table_exists?(:requests)
|
|
15
|
+
database.connection.create_table(:requests) do |t|
|
|
16
|
+
t.column :first_lineno, :integer
|
|
17
|
+
t.column :last_lineno, :integer
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
class RequestLogAnalyzer::Database::Source < RequestLogAnalyzer::Database::Base
|
|
2
|
+
|
|
3
|
+
def self.create_table!
|
|
4
|
+
unless database.connection.table_exists?(:sources)
|
|
5
|
+
database.connection.create_table(:sources) do |t|
|
|
6
|
+
t.column :filename, :string
|
|
7
|
+
t.column :mtime, :datetime
|
|
8
|
+
t.column :filesize, :integer
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
class RequestLogAnalyzer::Database::Warning < RequestLogAnalyzer::Database::Base
|
|
2
|
+
|
|
3
|
+
def self.create_table!
|
|
4
|
+
unless database.connection.table_exists?(:warnings)
|
|
5
|
+
database.connection.create_table(:warnings) do |t|
|
|
6
|
+
t.column :warning_type, :string, :limit => 30, :null => false
|
|
7
|
+
t.column :message, :string
|
|
8
|
+
t.column :source_id, :integer
|
|
9
|
+
t.column :lineno, :integer
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
end
|
|
@@ -5,13 +5,13 @@ class RequestLogAnalyzer::Database
|
|
|
5
5
|
|
|
6
6
|
def self.const_missing(const) # :nodoc:
|
|
7
7
|
RequestLogAnalyzer::load_default_class_file(self, const)
|
|
8
|
-
end
|
|
8
|
+
end
|
|
9
9
|
|
|
10
10
|
include RequestLogAnalyzer::Database::Connection
|
|
11
11
|
|
|
12
12
|
attr_accessor :file_format
|
|
13
|
-
attr_reader :
|
|
14
|
-
|
|
13
|
+
attr_reader :line_classes
|
|
14
|
+
|
|
15
15
|
def initialize(connection_identifier = nil)
|
|
16
16
|
@line_classes = []
|
|
17
17
|
RequestLogAnalyzer::Database::Base.database = self
|
|
@@ -23,112 +23,32 @@ class RequestLogAnalyzer::Database
|
|
|
23
23
|
line_type = line_type.name if line_type.respond_to?(:name)
|
|
24
24
|
Object.const_get("#{line_type}_line".camelize)
|
|
25
25
|
end
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
# It will create the class if not previously done so. The class will
|
|
30
|
-
# include a create_table! method the migrate the database.
|
|
31
|
-
def request_class
|
|
32
|
-
@request_class ||= begin
|
|
33
|
-
klass = Class.new(RequestLogAnalyzer::Database::Base) do
|
|
34
|
-
|
|
35
|
-
def lines
|
|
36
|
-
@lines ||= begin
|
|
37
|
-
lines = []
|
|
38
|
-
self.class.reflections.each { |r, d| lines += self.send(r).all }
|
|
39
|
-
lines.sort
|
|
40
|
-
end
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
# Creates the requests table
|
|
44
|
-
def self.create_table!
|
|
45
|
-
unless database.connection.table_exists?(:requests)
|
|
46
|
-
database.connection.create_table(:requests) do |t|
|
|
47
|
-
t.column :first_lineno, :integer
|
|
48
|
-
t.column :last_lineno, :integer
|
|
49
|
-
end
|
|
50
|
-
end
|
|
51
|
-
end
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
Object.const_set('Request', klass)
|
|
55
|
-
Object.const_get('Request')
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
# Returns the Source ORM class for the current database.
|
|
60
|
-
#
|
|
61
|
-
# It will create the class if not previously done so. The class will
|
|
62
|
-
# include a create_table! method the migrate the database.
|
|
63
|
-
def source_class
|
|
64
|
-
@source_class ||= begin
|
|
65
|
-
klass = Class.new(RequestLogAnalyzer::Database::Base) do
|
|
66
|
-
|
|
67
|
-
# Creates the sources table
|
|
68
|
-
def self.create_table!
|
|
69
|
-
unless database.connection.table_exists?(:sources)
|
|
70
|
-
database.connection.create_table(:sources) do |t|
|
|
71
|
-
t.column :filename, :string
|
|
72
|
-
t.column :mtime, :datetime
|
|
73
|
-
t.column :filesize, :integer
|
|
74
|
-
end
|
|
75
|
-
end
|
|
76
|
-
end
|
|
77
|
-
end
|
|
78
|
-
|
|
79
|
-
Object.const_set('Source', klass)
|
|
80
|
-
Object.const_get('Source')
|
|
81
|
-
end
|
|
82
|
-
end
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
# Returns the Warning ORM class for the current database.
|
|
86
|
-
#
|
|
87
|
-
# It will create the class if not previously done so. The class will
|
|
88
|
-
# include a create_table! method the migrate the database.
|
|
89
|
-
def warning_class
|
|
90
|
-
@warning_class ||= begin
|
|
91
|
-
klass = Class.new(RequestLogAnalyzer::Database::Base) do
|
|
92
|
-
|
|
93
|
-
# Creates the warnings table
|
|
94
|
-
def self.create_table!
|
|
95
|
-
unless database.connection.table_exists?(:warnings)
|
|
96
|
-
database.connection.create_table(:warnings) do |t|
|
|
97
|
-
t.column :warning_type, :string, :limit => 30, :null => false
|
|
98
|
-
t.column :message, :string
|
|
99
|
-
t.column :source_id, :integer
|
|
100
|
-
t.column :lineno, :integer
|
|
101
|
-
end
|
|
102
|
-
end
|
|
103
|
-
end
|
|
104
|
-
end
|
|
105
|
-
|
|
106
|
-
Object.const_set('Warning', klass)
|
|
107
|
-
Object.const_get('Warning')
|
|
108
|
-
end
|
|
26
|
+
|
|
27
|
+
def default_classes
|
|
28
|
+
[RequestLogAnalyzer::Database::Request, RequestLogAnalyzer::Database::Source, RequestLogAnalyzer::Database::Warning]
|
|
109
29
|
end
|
|
110
|
-
|
|
30
|
+
|
|
111
31
|
# Loads the ORM classes by inspecting the tables in the current database
|
|
112
32
|
def load_database_schema!
|
|
113
33
|
connection.tables.map do |table|
|
|
114
34
|
case table.to_sym
|
|
115
|
-
when :warnings then
|
|
116
|
-
when :sources then
|
|
117
|
-
when :requests then
|
|
35
|
+
when :warnings then RequestLogAnalyzer::Database::Warning
|
|
36
|
+
when :sources then RequestLogAnalyzer::Database::Source
|
|
37
|
+
when :requests then RequestLogAnalyzer::Database::Request
|
|
118
38
|
else load_activerecord_class(table)
|
|
119
39
|
end
|
|
120
40
|
end
|
|
121
41
|
end
|
|
122
|
-
|
|
42
|
+
|
|
123
43
|
# Returns an array of all the ActiveRecord-bases ORM classes for this database
|
|
124
44
|
def orm_classes
|
|
125
|
-
|
|
45
|
+
default_classes + line_classes
|
|
126
46
|
end
|
|
127
|
-
|
|
47
|
+
|
|
128
48
|
# Loads an ActiveRecord-based class that correspond to the given parameter, which can either be
|
|
129
49
|
# a table name or a LineDefinition instance.
|
|
130
50
|
def load_activerecord_class(linedefinition_or_table)
|
|
131
|
-
|
|
51
|
+
|
|
132
52
|
case linedefinition_or_table
|
|
133
53
|
when String, Symbol
|
|
134
54
|
klass_name = linedefinition_or_table.to_s.singularize.camelize
|
|
@@ -137,39 +57,44 @@ class RequestLogAnalyzer::Database
|
|
|
137
57
|
klass_name = "#{linedefinition_or_table.name}_line".camelize
|
|
138
58
|
klass = RequestLogAnalyzer::Database::Base.subclass_from_line_definition(linedefinition_or_table)
|
|
139
59
|
end
|
|
140
|
-
|
|
60
|
+
|
|
141
61
|
Object.const_set(klass_name, klass)
|
|
142
62
|
klass = Object.const_get(klass_name)
|
|
143
63
|
@line_classes << klass
|
|
144
64
|
return klass
|
|
145
|
-
end
|
|
65
|
+
end
|
|
146
66
|
|
|
147
67
|
def fileformat_classes
|
|
148
68
|
raise "No file_format provided!" unless file_format
|
|
149
|
-
|
|
150
|
-
default_classes = [request_class, source_class, warning_class]
|
|
151
69
|
line_classes = file_format.line_definitions.map { |(name, definition)| load_activerecord_class(definition) }
|
|
152
70
|
return default_classes + line_classes
|
|
153
71
|
end
|
|
154
72
|
|
|
155
|
-
# Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
|
|
73
|
+
# Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
|
|
156
74
|
# file format definition. These ORM classes will later be used to create records in the database.
|
|
157
75
|
def create_database_schema!
|
|
158
76
|
fileformat_classes.each { |klass| klass.create_table! }
|
|
159
77
|
end
|
|
160
|
-
|
|
78
|
+
|
|
161
79
|
# Drops the table of all the ORM classes, and unregisters the classes
|
|
162
80
|
def drop_database_schema!
|
|
163
81
|
file_format ? fileformat_classes.map(&:drop_table!) : orm_classes.map(&:drop_table!)
|
|
164
82
|
remove_orm_classes!
|
|
165
83
|
end
|
|
166
|
-
|
|
84
|
+
|
|
85
|
+
# Registers the default ORM classes in the default namespace
|
|
86
|
+
def register_default_orm_classes!
|
|
87
|
+
Object.const_set('Request', RequestLogAnalyzer::Database::Request)
|
|
88
|
+
Object.const_set('Source', RequestLogAnalyzer::Database::Source)
|
|
89
|
+
Object.const_set('Warning', RequestLogAnalyzer::Database::Warning)
|
|
90
|
+
end
|
|
91
|
+
|
|
167
92
|
# Unregisters every ORM class constant
|
|
168
93
|
def remove_orm_classes!
|
|
169
94
|
orm_classes.each do |klass|
|
|
170
95
|
if klass.respond_to?(:name) && !klass.name.blank?
|
|
171
|
-
|
|
172
|
-
Object.send(:remove_const,
|
|
96
|
+
klass_name = klass.name.split('::').last
|
|
97
|
+
Object.send(:remove_const, klass_name) if Object.const_defined?(klass_name)
|
|
173
98
|
end
|
|
174
99
|
end
|
|
175
100
|
end
|
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
|
2
2
|
|
|
3
|
-
# FileFormat for Amazon S3 access logs.
|
|
3
|
+
# FileFormat for Amazon S3 access logs.
|
|
4
4
|
#
|
|
5
5
|
# Access logs are disabled by default on Amazon S3. To enable logging, see
|
|
6
6
|
# http://docs.amazonwebservices.com/AmazonS3/latest/index.html?ServerLogs.html
|
|
7
7
|
class AmazonS3 < Base
|
|
8
|
-
|
|
8
|
+
|
|
9
9
|
line_definition :access do |line|
|
|
10
10
|
line.header = true
|
|
11
11
|
line.footer = true
|
|
12
12
|
line.regexp = /^([^\ ]+) ([^\ ]+) \[(\d{2}\/[A-Za-z]{3}\/\d{4}.\d{2}:\d{2}:\d{2})(?: .\d{4})?\] (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]+)" "([^"]+)"/
|
|
13
|
-
line.captures << { :name => :bucket_owner, :type => :string } <<
|
|
14
|
-
{ :name => :bucket, :type => :string } <<
|
|
13
|
+
line.captures << { :name => :bucket_owner, :type => :string } <<
|
|
14
|
+
{ :name => :bucket, :type => :string } <<
|
|
15
15
|
{ :name => :timestamp, :type => :timestamp } <<
|
|
16
16
|
{ :name => :remote_ip, :type => :string } <<
|
|
17
|
-
{ :name => :requester, :type => :string } <<
|
|
17
|
+
{ :name => :requester, :type => :string } <<
|
|
18
18
|
{ :name => :request_id, :type => :string } <<
|
|
19
19
|
{ :name => :operation, :type => :string } <<
|
|
20
20
|
{ :name => :key, :type => :nillable_string } <<
|
|
@@ -28,45 +28,44 @@ module RequestLogAnalyzer::FileFormat
|
|
|
28
28
|
{ :name => :referer, :type => :referer } <<
|
|
29
29
|
{ :name => :user_agent, :type => :user_agent }
|
|
30
30
|
end
|
|
31
|
-
|
|
31
|
+
|
|
32
32
|
report do |analyze|
|
|
33
33
|
analyze.timespan
|
|
34
34
|
analyze.hourly_spread
|
|
35
35
|
|
|
36
|
-
analyze.frequency :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :
|
|
37
|
-
analyze.duration :duration => :total_time, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :
|
|
38
|
-
analyze.traffic :traffic => :bytes_sent, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :
|
|
36
|
+
analyze.frequency :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Most popular items"
|
|
37
|
+
analyze.duration :duration => :total_time, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Request duration"
|
|
38
|
+
analyze.traffic :traffic => :bytes_sent, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Traffic"
|
|
39
39
|
analyze.frequency :category => :http_status, :title => 'HTTP status codes'
|
|
40
40
|
analyze.frequency :category => :error_code, :title => 'Error codes'
|
|
41
41
|
end
|
|
42
|
-
|
|
42
|
+
|
|
43
43
|
class Request < RequestLogAnalyzer::Request
|
|
44
|
-
|
|
44
|
+
|
|
45
45
|
MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
|
|
46
46
|
'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
|
|
47
|
-
|
|
47
|
+
|
|
48
48
|
# Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
|
|
49
49
|
# to speed up parsing.
|
|
50
50
|
def convert_timestamp(value, definition)
|
|
51
|
-
|
|
52
|
-
"#{d[2]}#{MONTHS[d[1]]}#{d[0]}#{d[3]}#{d[4]}#{d[5]}".to_i
|
|
51
|
+
"#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
|
|
53
52
|
end
|
|
54
|
-
|
|
53
|
+
|
|
55
54
|
# Make sure that the string '-' is parsed as a nil value.
|
|
56
55
|
def convert_nillable_string(value, definition)
|
|
57
56
|
value == '-' ? nil : value
|
|
58
57
|
end
|
|
59
|
-
|
|
58
|
+
|
|
60
59
|
# Can be implemented in subclasses for improved categorizations
|
|
61
60
|
def convert_referer(value, definition)
|
|
62
61
|
value == '-' ? nil : value
|
|
63
62
|
end
|
|
64
|
-
|
|
63
|
+
|
|
65
64
|
# Can be implemented in subclasses for improved categorizations
|
|
66
65
|
def convert_user_agent(value, definition)
|
|
67
66
|
value == '-' ? nil : value
|
|
68
67
|
end
|
|
69
68
|
end
|
|
70
|
-
|
|
69
|
+
|
|
71
70
|
end
|
|
72
71
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
|
2
|
-
|
|
3
|
-
# The Apache file format is able to log Apache access.log files.
|
|
2
|
+
|
|
3
|
+
# The Apache file format is able to log Apache access.log files.
|
|
4
4
|
#
|
|
5
5
|
# The access.log can be configured in Apache to have many different formats. In theory, this
|
|
6
6
|
# FileFormat can handle any format, but it must be aware of the log formatting that is used
|
|
@@ -22,7 +22,7 @@ module RequestLogAnalyzer::FileFormat
|
|
|
22
22
|
:referer => '%{Referer}i -> %U',
|
|
23
23
|
:agent => '%{User-agent}i'
|
|
24
24
|
}
|
|
25
|
-
|
|
25
|
+
|
|
26
26
|
# A hash that defines how the log format directives should be parsed.
|
|
27
27
|
LOG_DIRECTIVES = {
|
|
28
28
|
'%' => { :regexp => '%', :captures => [] },
|
|
@@ -48,8 +48,9 @@ module RequestLogAnalyzer::FileFormat
|
|
|
48
48
|
# It will set up the line definition and the report trackers according to the Apache access log format,
|
|
49
49
|
# which should be passed as first argument. By default, is uses the 'combined' log format.
|
|
50
50
|
def self.create(*args)
|
|
51
|
-
access_line =
|
|
52
|
-
|
|
51
|
+
access_line = access_line_definition(args.first)
|
|
52
|
+
trackers = report_trackers(access_line) + report_definer.trackers
|
|
53
|
+
self.new(line_definer.line_definitions.merge(:access => access_line), trackers)
|
|
53
54
|
end
|
|
54
55
|
|
|
55
56
|
# Creates the access log line definition based on the Apache log format string
|
|
@@ -62,7 +63,7 @@ module RequestLogAnalyzer::FileFormat
|
|
|
62
63
|
format_string.scan(/([^%]*)(?:%(?:\{([^\}]+)\})?>?([A-Za-z%]))?/) do |literal, arg, variable|
|
|
63
64
|
|
|
64
65
|
line_regexp << Regexp.quote(literal) # Make sure to parse the literal before the directive
|
|
65
|
-
|
|
66
|
+
|
|
66
67
|
if variable
|
|
67
68
|
# Check if we recognize the log directive
|
|
68
69
|
directive = LOG_DIRECTIVES[variable]
|
|
@@ -77,7 +78,7 @@ module RequestLogAnalyzer::FileFormat
|
|
|
77
78
|
end
|
|
78
79
|
end
|
|
79
80
|
end
|
|
80
|
-
|
|
81
|
+
|
|
81
82
|
# Return a new line definition object
|
|
82
83
|
return RequestLogAnalyzer::LineDefinition.new(:access, :regexp => Regexp.new(line_regexp),
|
|
83
84
|
:captures => captures, :header => true, :footer => true)
|
|
@@ -90,49 +91,47 @@ module RequestLogAnalyzer::FileFormat
|
|
|
90
91
|
analyze.timespan if line_definition.captures?(:timestamp)
|
|
91
92
|
analyze.hourly_spread if line_definition.captures?(:timestamp)
|
|
92
93
|
|
|
93
|
-
analyze.frequency :category => :http_method, :
|
|
94
|
-
analyze.frequency :category => :http_status, :
|
|
95
|
-
analyze.frequency :category =>
|
|
94
|
+
analyze.frequency :category => :http_method, :title => "HTTP methods" if line_definition.captures?(:http_method)
|
|
95
|
+
analyze.frequency :category => :http_status, :title => "HTTP statuses" if line_definition.captures?(:http_status)
|
|
96
|
+
analyze.frequency :category => lambda { |r| r.category }, :title => "Most popular URIs" if line_definition.captures?(:path)
|
|
96
97
|
|
|
97
|
-
analyze.frequency :category => :user_agent, :
|
|
98
|
-
analyze.frequency :category => :referer, :
|
|
98
|
+
analyze.frequency :category => :user_agent, :title => "User agents" if line_definition.captures?(:user_agent)
|
|
99
|
+
analyze.frequency :category => :referer, :title => "Referers" if line_definition.captures?(:referer)
|
|
99
100
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
end
|
|
103
|
-
|
|
104
|
-
if line_definition.captures?(:path) && line_definition.captures?(:bytes_sent)
|
|
105
|
-
analyze.traffic :traffic => :bytes_sent, :category => :path , :title => 'Traffic'
|
|
106
|
-
end
|
|
101
|
+
analyze.duration :duration => :duration, :category => lambda { |r| r.category }, :title => 'Request duration' if line_definition.captures?(:duration)
|
|
102
|
+
analyze.traffic :traffic => :bytes_sent, :category => lambda { |r| r.category }, :title => 'Traffic' if line_definition.captures?(:bytes_sent)
|
|
107
103
|
|
|
108
104
|
return analyze.trackers
|
|
109
105
|
end
|
|
110
106
|
|
|
111
107
|
# Define a custom Request class for the Apache file format to speed up timestamp handling.
|
|
112
108
|
class Request < RequestLogAnalyzer::Request
|
|
113
|
-
|
|
109
|
+
|
|
110
|
+
def category
|
|
111
|
+
first(:path)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
114
|
MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
|
|
115
115
|
'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
|
|
116
|
-
|
|
116
|
+
|
|
117
117
|
# Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
|
|
118
118
|
# to speed up parsing.
|
|
119
119
|
def convert_timestamp(value, definition)
|
|
120
|
-
|
|
121
|
-
"#{d[2]}#{MONTHS[d[1]]}#{d[0]}#{d[3]}#{d[4]}#{d[5]}".to_i
|
|
120
|
+
"#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
|
|
122
121
|
end
|
|
123
|
-
|
|
122
|
+
|
|
124
123
|
# This function can be overridden to rewrite the path for better categorization in the
|
|
125
124
|
# reports.
|
|
126
125
|
def convert_path(value, definition)
|
|
127
126
|
value
|
|
128
127
|
end
|
|
129
|
-
|
|
130
|
-
# This function can be overridden to simplify the user agent string for better
|
|
128
|
+
|
|
129
|
+
# This function can be overridden to simplify the user agent string for better
|
|
131
130
|
# categorization in the reports
|
|
132
131
|
def convert_user_agent(value, definition)
|
|
133
132
|
value # TODO
|
|
134
133
|
end
|
|
135
|
-
|
|
134
|
+
|
|
136
135
|
# Make sure that the string '-' is parsed as a nil value.
|
|
137
136
|
def convert_nillable_string(value, definition)
|
|
138
137
|
value == '-' ? nil : value
|
|
@@ -1,50 +1,67 @@
|
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
|
2
|
-
|
|
2
|
+
|
|
3
|
+
# The Merb file format parses the request header with the timestamp, the params line
|
|
4
|
+
# with the most important request information and the durations line which contains
|
|
5
|
+
# the different request durations that can be used for analysis.
|
|
3
6
|
class Merb < Base
|
|
4
7
|
|
|
5
8
|
# ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
|
|
6
9
|
line_definition :started do |line|
|
|
7
10
|
line.header = true
|
|
8
|
-
line.teaser = /Started
|
|
11
|
+
line.teaser = /Started request handling\:/
|
|
9
12
|
line.regexp = /Started request handling\:\ (.+)/
|
|
10
13
|
line.captures << { :name => :timestamp, :type => :timestamp }
|
|
11
|
-
end
|
|
12
|
-
|
|
14
|
+
end
|
|
15
|
+
|
|
13
16
|
# ~ Params: {"action"=>"create", "controller"=>"session"}
|
|
14
17
|
# ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
|
|
15
18
|
line_definition :params do |line|
|
|
16
|
-
line.teaser = /Params/
|
|
19
|
+
line.teaser = /Params\:\ /
|
|
17
20
|
line.regexp = /Params\:\ (\{.+\})/
|
|
18
|
-
line.captures << { :name => :params, :type => :eval, :provides => {
|
|
21
|
+
line.captures << { :name => :params, :type => :eval, :provides => {
|
|
19
22
|
:namespace => :string, :controller => :string, :action => :string, :format => :string, :method => :string } }
|
|
20
23
|
end
|
|
21
24
|
|
|
22
25
|
# ~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
|
|
23
26
|
line_definition :completed do |line|
|
|
24
27
|
line.footer = true
|
|
28
|
+
# line.teaser = Regexp.new(Regexp.quote('~ {:'))
|
|
25
29
|
line.regexp = /(\{.*\:dispatch_time\s*=>\s*\d+\.\d+.*\})/
|
|
26
30
|
line.captures << { :name => :times_hash, :type => :eval, :provides => {
|
|
27
31
|
:dispatch_time => :duration, :after_filters_time => :duration,
|
|
28
32
|
:before_filters_time => :duration, :action_time => :duration } }
|
|
29
33
|
end
|
|
30
|
-
|
|
31
|
-
REQUEST_CATEGORIZER = Proc.new do |request|
|
|
34
|
+
|
|
35
|
+
REQUEST_CATEGORIZER = Proc.new do |request|
|
|
32
36
|
category = "#{request[:controller]}##{request[:action]}"
|
|
33
37
|
category = "#{request[:namespace]}::#{category}" if request[:namespace]
|
|
34
38
|
category = "#{category}.#{request[:format]}" if request[:format]
|
|
35
39
|
category
|
|
36
40
|
end
|
|
37
|
-
|
|
41
|
+
|
|
38
42
|
report do |analyze|
|
|
39
|
-
|
|
40
|
-
analyze.
|
|
41
|
-
analyze.hourly_spread
|
|
43
|
+
|
|
44
|
+
analyze.timespan
|
|
45
|
+
analyze.hourly_spread
|
|
46
|
+
|
|
47
|
+
analyze.frequency :category => REQUEST_CATEGORIZER, :title => "Top 20 by hits"
|
|
42
48
|
analyze.duration :dispatch_time, :category => REQUEST_CATEGORIZER, :title => 'Request dispatch duration'
|
|
49
|
+
|
|
43
50
|
# analyze.duration :action_time, :category => REQUEST_CATEGORIZER, :title => 'Request action duration'
|
|
44
51
|
# analyze.duration :after_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request after_filter duration'
|
|
45
52
|
# analyze.duration :before_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request before_filter duration'
|
|
46
53
|
end
|
|
47
|
-
|
|
54
|
+
|
|
55
|
+
class Request < RequestLogAnalyzer::Request
|
|
56
|
+
|
|
57
|
+
MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
|
|
58
|
+
'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
|
|
59
|
+
|
|
60
|
+
# Speed up timestamp conversion
|
|
61
|
+
def convert_timestamp(value, definition)
|
|
62
|
+
"#{value[26,4]}#{MONTHS[value[4,3]]}#{value[8,2]}#{value[11,2]}#{value[14,2]}#{value[17,2]}".to_i
|
|
63
|
+
end
|
|
64
|
+
end
|
|
48
65
|
end
|
|
49
66
|
|
|
50
67
|
end
|