request-log-analyzer 1.3.4 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. data/LICENSE +3 -3
  2. data/README.rdoc +1 -1
  3. data/bin/request-log-analyzer +17 -14
  4. data/lib/cli/command_line_arguments.rb +51 -51
  5. data/lib/cli/database_console.rb +3 -3
  6. data/lib/cli/database_console_init.rb +4 -3
  7. data/lib/cli/progressbar.rb +10 -10
  8. data/lib/cli/tools.rb +3 -3
  9. data/lib/request_log_analyzer/aggregator/database_inserter.rb +19 -17
  10. data/lib/request_log_analyzer/aggregator/echo.rb +14 -9
  11. data/lib/request_log_analyzer/aggregator/summarizer.rb +26 -26
  12. data/lib/request_log_analyzer/aggregator.rb +11 -15
  13. data/lib/request_log_analyzer/controller.rb +162 -89
  14. data/lib/request_log_analyzer/database/base.rb +22 -21
  15. data/lib/request_log_analyzer/database/connection.rb +3 -3
  16. data/lib/request_log_analyzer/database/request.rb +22 -0
  17. data/lib/request_log_analyzer/database/source.rb +13 -0
  18. data/lib/request_log_analyzer/database/warning.rb +14 -0
  19. data/lib/request_log_analyzer/database.rb +28 -103
  20. data/lib/request_log_analyzer/file_format/amazon_s3.rb +20 -20
  21. data/lib/request_log_analyzer/file_format/apache.rb +37 -30
  22. data/lib/request_log_analyzer/file_format/merb.rb +30 -13
  23. data/lib/request_log_analyzer/file_format/rack.rb +11 -0
  24. data/lib/request_log_analyzer/file_format/rails.rb +143 -73
  25. data/lib/request_log_analyzer/file_format/rails_development.rb +4 -49
  26. data/lib/request_log_analyzer/file_format.rb +16 -28
  27. data/lib/request_log_analyzer/filter/anonymize.rb +8 -7
  28. data/lib/request_log_analyzer/filter/field.rb +9 -9
  29. data/lib/request_log_analyzer/filter/timespan.rb +12 -10
  30. data/lib/request_log_analyzer/filter.rb +12 -16
  31. data/lib/request_log_analyzer/line_definition.rb +15 -14
  32. data/lib/request_log_analyzer/log_processor.rb +27 -29
  33. data/lib/request_log_analyzer/mailer.rb +15 -9
  34. data/lib/request_log_analyzer/output/fixed_width.rb +48 -48
  35. data/lib/request_log_analyzer/output/html.rb +20 -20
  36. data/lib/request_log_analyzer/output.rb +53 -12
  37. data/lib/request_log_analyzer/request.rb +88 -69
  38. data/lib/request_log_analyzer/source/database_loader.rb +10 -14
  39. data/lib/request_log_analyzer/source/log_parser.rb +57 -50
  40. data/lib/request_log_analyzer/source.rb +10 -12
  41. data/lib/request_log_analyzer/tracker/duration.rb +41 -134
  42. data/lib/request_log_analyzer/tracker/frequency.rb +31 -32
  43. data/lib/request_log_analyzer/tracker/hourly_spread.rb +21 -21
  44. data/lib/request_log_analyzer/tracker/timespan.rb +17 -17
  45. data/lib/request_log_analyzer/tracker/traffic.rb +106 -0
  46. data/lib/request_log_analyzer/tracker.rb +139 -32
  47. data/lib/request_log_analyzer.rb +5 -5
  48. data/request-log-analyzer.gemspec +19 -15
  49. data/spec/database.yml +6 -0
  50. data/spec/fixtures/rails_22.log +1 -1
  51. data/spec/integration/command_line_usage_spec.rb +7 -1
  52. data/spec/lib/helpers.rb +7 -7
  53. data/spec/lib/macros.rb +3 -3
  54. data/spec/lib/matchers.rb +41 -27
  55. data/spec/lib/mocks.rb +19 -15
  56. data/spec/lib/testing_format.rb +9 -9
  57. data/spec/spec_helper.rb +6 -6
  58. data/spec/unit/aggregator/database_inserter_spec.rb +16 -16
  59. data/spec/unit/aggregator/summarizer_spec.rb +4 -4
  60. data/spec/unit/controller/controller_spec.rb +2 -2
  61. data/spec/unit/controller/log_processor_spec.rb +1 -1
  62. data/spec/unit/database/base_class_spec.rb +26 -33
  63. data/spec/unit/database/connection_spec.rb +3 -3
  64. data/spec/unit/database/database_spec.rb +33 -38
  65. data/spec/unit/file_format/amazon_s3_format_spec.rb +5 -5
  66. data/spec/unit/file_format/apache_format_spec.rb +48 -11
  67. data/spec/unit/file_format/file_format_api_spec.rb +13 -13
  68. data/spec/unit/file_format/line_definition_spec.rb +24 -17
  69. data/spec/unit/file_format/merb_format_spec.rb +41 -45
  70. data/spec/unit/file_format/rails_format_spec.rb +157 -117
  71. data/spec/unit/filter/anonymize_filter_spec.rb +2 -2
  72. data/spec/unit/filter/field_filter_spec.rb +13 -13
  73. data/spec/unit/filter/filter_spec.rb +1 -1
  74. data/spec/unit/filter/timespan_filter_spec.rb +15 -15
  75. data/spec/unit/mailer_spec.rb +30 -0
  76. data/spec/unit/{source/request_spec.rb → request_spec.rb} +30 -30
  77. data/spec/unit/source/log_parser_spec.rb +27 -27
  78. data/spec/unit/tracker/duration_tracker_spec.rb +115 -78
  79. data/spec/unit/tracker/frequency_tracker_spec.rb +74 -63
  80. data/spec/unit/tracker/hourly_spread_spec.rb +28 -20
  81. data/spec/unit/tracker/timespan_tracker_spec.rb +25 -13
  82. data/spec/unit/tracker/tracker_api_spec.rb +117 -42
  83. data/spec/unit/tracker/traffic_tracker_spec.rb +107 -0
  84. data/tasks/github-gem.rake +125 -75
  85. data/tasks/request_log_analyzer.rake +2 -2
  86. metadata +19 -10
@@ -1,5 +1,5 @@
1
1
  class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
2
-
2
+
3
3
  self.abstract_class = true
4
4
 
5
5
  def <=>(other)
@@ -14,25 +14,26 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
14
14
  self.class.name.underscore.gsub(/_line$/, '').to_sym
15
15
  end
16
16
 
17
- cattr_accessor :database, :line_definition
17
+ class_inheritable_accessor :line_definition
18
+ cattr_accessor :database
18
19
 
19
20
  def self.subclass_from_line_definition(definition)
20
21
  klass = Class.new(RequestLogAnalyzer::Database::Base)
21
22
  klass.set_table_name("#{definition.name}_lines")
22
-
23
+
23
24
  klass.line_definition = definition
24
-
25
+
25
26
  # Set relations with requests and sources table
26
27
  klass.belongs_to :request
27
28
  klass.belongs_to :source
28
-
29
+
29
30
  # Serialize complex fields into the database
30
31
  definition.captures.select { |c| c.has_key?(:provides) }.each do |capture|
31
32
  klass.send(:serialize, capture[:name], Hash)
32
33
  end
33
34
 
34
- database.request_class.has_many "#{definition.name}_lines".to_sym
35
- database.source_class.has_many "#{definition.name}_lines".to_sym
35
+ RequestLogAnalyzer::Database::Request.has_many "#{definition.name}_lines".to_sym
36
+ RequestLogAnalyzer::Database::Source.has_many "#{definition.name}_lines".to_sym
36
37
 
37
38
  return klass
38
39
  end
@@ -45,34 +46,34 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
45
46
 
46
47
  if klass.column_names.include?('request_id')
47
48
  klass.belongs_to :request
48
- database.request_class.has_many table.to_sym
49
+ RequestLogAnalyzer::Database::Request.has_many table.to_sym
49
50
  end
50
-
51
+
51
52
  if klass.column_names.include?('source_id')
52
53
  klass.belongs_to :source
53
- database.source_class.has_many table.to_sym
54
+ RequestLogAnalyzer::Database::Source.has_many table.to_sym
54
55
  end
55
-
56
+
56
57
  return klass
57
58
  end
58
-
59
+
59
60
  def self.drop_table!
60
61
  database.connection.remove_index(self.table_name, [:source_id]) rescue nil
61
62
  database.connection.remove_index(self.table_name, [:request_id]) rescue nil
62
63
  database.connection.drop_table(self.table_name) if database.connection.table_exists?(self.table_name)
63
64
  end
64
-
65
+
65
66
  def self.create_table!
66
67
  raise "No line_definition available to base table schema on!" unless self.line_definition
67
-
68
+
68
69
  unless table_exists?
69
70
  database.connection.create_table(table_name.to_sym) do |t|
70
-
71
+
71
72
  # Default fields
72
73
  t.column :request_id, :integer
73
74
  t.column :source_id, :integer
74
75
  t.column :lineno, :integer
75
-
76
+
76
77
  line_definition.captures.each do |capture|
77
78
  # Add a field for every capture
78
79
  t.column(capture[:name], column_type(capture[:type]))
@@ -82,13 +83,13 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
82
83
  end
83
84
  end
84
85
  end
85
-
86
+
86
87
  # Add indices to table for more speedy querying
87
88
  database.connection.add_index(self.table_name.to_sym, [:request_id]) # rescue
88
89
  database.connection.add_index(self.table_name.to_sym, [:source_id]) # rescue
89
90
  end
90
-
91
-
91
+
92
+
92
93
  # Function to determine the column type for a field
93
94
  # TODO: make more robust / include in file-format definition
94
95
  def self.column_type(type_indicator)
@@ -109,6 +110,6 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
109
110
  when :date; :date
110
111
  else :string
111
112
  end
112
- end
113
-
113
+ end
114
+
114
115
  end
@@ -26,13 +26,13 @@ module RequestLogAnalyzer::Database::Connection
26
26
  raise "Cannot connect with this connection_identifier: #{connection_identifier.inspect}"
27
27
  end
28
28
  end
29
-
29
+
30
30
  def disconnect
31
31
  RequestLogAnalyzer::Database::Base.remove_connection
32
32
  end
33
-
33
+
34
34
  def connection
35
35
  RequestLogAnalyzer::Database::Base.connection
36
36
  end
37
-
37
+
38
38
  end
@@ -0,0 +1,22 @@
1
+ class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base
2
+
3
+ # Returns an array of all the Line objects of this request in the correct order.
4
+ def lines
5
+ @lines ||= begin
6
+ lines = []
7
+ self.class.reflections.each { |r, d| lines += self.send(r).all }
8
+ lines.sort
9
+ end
10
+ end
11
+
12
+ # Creates the table to store requests in.
13
+ def self.create_table!
14
+ unless database.connection.table_exists?(:requests)
15
+ database.connection.create_table(:requests) do |t|
16
+ t.column :first_lineno, :integer
17
+ t.column :last_lineno, :integer
18
+ end
19
+ end
20
+ end
21
+
22
+ end
@@ -0,0 +1,13 @@
1
+ class RequestLogAnalyzer::Database::Source < RequestLogAnalyzer::Database::Base
2
+
3
+ def self.create_table!
4
+ unless database.connection.table_exists?(:sources)
5
+ database.connection.create_table(:sources) do |t|
6
+ t.column :filename, :string
7
+ t.column :mtime, :datetime
8
+ t.column :filesize, :integer
9
+ end
10
+ end
11
+ end
12
+
13
+ end
@@ -0,0 +1,14 @@
1
+ class RequestLogAnalyzer::Database::Warning < RequestLogAnalyzer::Database::Base
2
+
3
+ def self.create_table!
4
+ unless database.connection.table_exists?(:warnings)
5
+ database.connection.create_table(:warnings) do |t|
6
+ t.column :warning_type, :string, :limit => 30, :null => false
7
+ t.column :message, :string
8
+ t.column :source_id, :integer
9
+ t.column :lineno, :integer
10
+ end
11
+ end
12
+ end
13
+
14
+ end
@@ -5,13 +5,13 @@ class RequestLogAnalyzer::Database
5
5
 
6
6
  def self.const_missing(const) # :nodoc:
7
7
  RequestLogAnalyzer::load_default_class_file(self, const)
8
- end
8
+ end
9
9
 
10
10
  include RequestLogAnalyzer::Database::Connection
11
11
 
12
12
  attr_accessor :file_format
13
- attr_reader :request_class, :warning_class, :source_class, :line_classes
14
-
13
+ attr_reader :line_classes
14
+
15
15
  def initialize(connection_identifier = nil)
16
16
  @line_classes = []
17
17
  RequestLogAnalyzer::Database::Base.database = self
@@ -23,112 +23,32 @@ class RequestLogAnalyzer::Database
23
23
  line_type = line_type.name if line_type.respond_to?(:name)
24
24
  Object.const_get("#{line_type}_line".camelize)
25
25
  end
26
-
27
- # Returns the Request ORM class for the current database.
28
- #
29
- # It will create the class if not previously done so. The class will
30
- # include a create_table! method the migrate the database.
31
- def request_class
32
- @request_class ||= begin
33
- klass = Class.new(RequestLogAnalyzer::Database::Base) do
34
-
35
- def lines
36
- @lines ||= begin
37
- lines = []
38
- self.class.reflections.each { |r, d| lines += self.send(r).all }
39
- lines.sort
40
- end
41
- end
42
-
43
- # Creates the requests table
44
- def self.create_table!
45
- unless database.connection.table_exists?(:requests)
46
- database.connection.create_table(:requests) do |t|
47
- t.column :first_lineno, :integer
48
- t.column :last_lineno, :integer
49
- end
50
- end
51
- end
52
- end
53
-
54
- Object.const_set('Request', klass)
55
- Object.const_get('Request')
56
- end
57
- end
58
-
59
- # Returns the Source ORM class for the current database.
60
- #
61
- # It will create the class if not previously done so. The class will
62
- # include a create_table! method the migrate the database.
63
- def source_class
64
- @source_class ||= begin
65
- klass = Class.new(RequestLogAnalyzer::Database::Base) do
66
-
67
- # Creates the sources table
68
- def self.create_table!
69
- unless database.connection.table_exists?(:sources)
70
- database.connection.create_table(:sources) do |t|
71
- t.column :filename, :string
72
- t.column :mtime, :datetime
73
- t.column :filesize, :integer
74
- end
75
- end
76
- end
77
- end
78
-
79
- Object.const_set('Source', klass)
80
- Object.const_get('Source')
81
- end
82
- end
83
-
84
-
85
- # Returns the Warning ORM class for the current database.
86
- #
87
- # It will create the class if not previously done so. The class will
88
- # include a create_table! method the migrate the database.
89
- def warning_class
90
- @warning_class ||= begin
91
- klass = Class.new(RequestLogAnalyzer::Database::Base) do
92
-
93
- # Creates the warnings table
94
- def self.create_table!
95
- unless database.connection.table_exists?(:warnings)
96
- database.connection.create_table(:warnings) do |t|
97
- t.column :warning_type, :string, :limit => 30, :null => false
98
- t.column :message, :string
99
- t.column :source_id, :integer
100
- t.column :lineno, :integer
101
- end
102
- end
103
- end
104
- end
105
-
106
- Object.const_set('Warning', klass)
107
- Object.const_get('Warning')
108
- end
26
+
27
+ def default_classes
28
+ [RequestLogAnalyzer::Database::Request, RequestLogAnalyzer::Database::Source, RequestLogAnalyzer::Database::Warning]
109
29
  end
110
-
30
+
111
31
  # Loads the ORM classes by inspecting the tables in the current database
112
32
  def load_database_schema!
113
33
  connection.tables.map do |table|
114
34
  case table.to_sym
115
- when :warnings then warning_class
116
- when :sources then source_class
117
- when :requests then request_class
35
+ when :warnings then RequestLogAnalyzer::Database::Warning
36
+ when :sources then RequestLogAnalyzer::Database::Source
37
+ when :requests then RequestLogAnalyzer::Database::Request
118
38
  else load_activerecord_class(table)
119
39
  end
120
40
  end
121
41
  end
122
-
42
+
123
43
  # Returns an array of all the ActiveRecord-bases ORM classes for this database
124
44
  def orm_classes
125
- [warning_class, request_class, source_class] + line_classes
45
+ default_classes + line_classes
126
46
  end
127
-
47
+
128
48
  # Loads an ActiveRecord-based class that correspond to the given parameter, which can either be
129
49
  # a table name or a LineDefinition instance.
130
50
  def load_activerecord_class(linedefinition_or_table)
131
-
51
+
132
52
  case linedefinition_or_table
133
53
  when String, Symbol
134
54
  klass_name = linedefinition_or_table.to_s.singularize.camelize
@@ -137,39 +57,44 @@ class RequestLogAnalyzer::Database
137
57
  klass_name = "#{linedefinition_or_table.name}_line".camelize
138
58
  klass = RequestLogAnalyzer::Database::Base.subclass_from_line_definition(linedefinition_or_table)
139
59
  end
140
-
60
+
141
61
  Object.const_set(klass_name, klass)
142
62
  klass = Object.const_get(klass_name)
143
63
  @line_classes << klass
144
64
  return klass
145
- end
65
+ end
146
66
 
147
67
  def fileformat_classes
148
68
  raise "No file_format provided!" unless file_format
149
-
150
- default_classes = [request_class, source_class, warning_class]
151
69
  line_classes = file_format.line_definitions.map { |(name, definition)| load_activerecord_class(definition) }
152
70
  return default_classes + line_classes
153
71
  end
154
72
 
155
- # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
73
+ # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
156
74
  # file format definition. These ORM classes will later be used to create records in the database.
157
75
  def create_database_schema!
158
76
  fileformat_classes.each { |klass| klass.create_table! }
159
77
  end
160
-
78
+
161
79
  # Drops the table of all the ORM classes, and unregisters the classes
162
80
  def drop_database_schema!
163
81
  file_format ? fileformat_classes.map(&:drop_table!) : orm_classes.map(&:drop_table!)
164
82
  remove_orm_classes!
165
83
  end
166
-
84
+
85
+ # Registers the default ORM classes in the default namespace
86
+ def register_default_orm_classes!
87
+ Object.const_set('Request', RequestLogAnalyzer::Database::Request)
88
+ Object.const_set('Source', RequestLogAnalyzer::Database::Source)
89
+ Object.const_set('Warning', RequestLogAnalyzer::Database::Warning)
90
+ end
91
+
167
92
  # Unregisters every ORM class constant
168
93
  def remove_orm_classes!
169
94
  orm_classes.each do |klass|
170
95
  if klass.respond_to?(:name) && !klass.name.blank?
171
- # klass_base_name = klass.name.split('::').last
172
- Object.send(:remove_const, klass.name) if Object.const_defined?(klass.name)
96
+ klass_name = klass.name.split('::').last
97
+ Object.send(:remove_const, klass_name) if Object.const_defined?(klass_name)
173
98
  end
174
99
  end
175
100
  end
@@ -1,71 +1,71 @@
1
1
  module RequestLogAnalyzer::FileFormat
2
2
 
3
- # FileFormat for Amazon S3 access logs.
3
+ # FileFormat for Amazon S3 access logs.
4
4
  #
5
5
  # Access logs are disabled by default on Amazon S3. To enable logging, see
6
6
  # http://docs.amazonwebservices.com/AmazonS3/latest/index.html?ServerLogs.html
7
7
  class AmazonS3 < Base
8
-
8
+
9
9
  line_definition :access do |line|
10
10
  line.header = true
11
11
  line.footer = true
12
- line.regexp = /^([^\ ]+) ([^\ ]+) \[([^\]]{26})\] (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]+)" "([^"]+)"/
13
- line.captures << { :name => :bucket_owner, :type => :string } <<
14
- { :name => :bucket, :type => :string } <<
12
+ line.regexp = /^([^\ ]+) ([^\ ]+) \[(\d{2}\/[A-Za-z]{3}\/\d{4}.\d{2}:\d{2}:\d{2})(?: .\d{4})?\] (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]+)" "([^"]+)"/
13
+ line.captures << { :name => :bucket_owner, :type => :string } <<
14
+ { :name => :bucket, :type => :string } <<
15
15
  { :name => :timestamp, :type => :timestamp } <<
16
16
  { :name => :remote_ip, :type => :string } <<
17
- { :name => :requester, :type => :string } <<
17
+ { :name => :requester, :type => :string } <<
18
18
  { :name => :request_id, :type => :string } <<
19
19
  { :name => :operation, :type => :string } <<
20
20
  { :name => :key, :type => :nillable_string } <<
21
21
  { :name => :request_uri, :type => :string } <<
22
22
  { :name => :http_status, :type => :integer } <<
23
23
  { :name => :error_code, :type => :nillable_string } <<
24
- { :name => :bytes_sent, :type => :integer } <<
25
- { :name => :object_size, :type => :integer } <<
24
+ { :name => :bytes_sent, :type => :traffic, :unit => :byte } <<
25
+ { :name => :object_size, :type => :traffic, :unit => :byte } <<
26
26
  { :name => :total_time, :type => :duration, :unit => :msec } <<
27
27
  { :name => :turnaround_time, :type => :duration, :unit => :msec } <<
28
28
  { :name => :referer, :type => :referer } <<
29
29
  { :name => :user_agent, :type => :user_agent }
30
30
  end
31
-
31
+
32
32
  report do |analyze|
33
33
  analyze.timespan
34
34
  analyze.hourly_spread
35
35
 
36
- analyze.frequency :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :amount => 20, :title => "Most popular items"
37
- analyze.duration :duration => :total_time, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :amount => 20, :title => "Duration"
36
+ analyze.frequency :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Most popular items"
37
+ analyze.duration :duration => :total_time, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Request duration"
38
+ analyze.traffic :traffic => :bytes_sent, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Traffic"
38
39
  analyze.frequency :category => :http_status, :title => 'HTTP status codes'
39
40
  analyze.frequency :category => :error_code, :title => 'Error codes'
40
41
  end
41
-
42
+
42
43
  class Request < RequestLogAnalyzer::Request
43
-
44
+
44
45
  MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
45
46
  'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
46
-
47
+
47
48
  # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
48
49
  # to speed up parsing.
49
50
  def convert_timestamp(value, definition)
50
- d = /^(\d{2})\/(\w{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2})/.match(value).captures
51
- "#{d[2]}#{MONTHS[d[1]]}#{d[0]}#{d[3]}#{d[4]}#{d[5]}".to_i
51
+ "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
52
52
  end
53
-
53
+
54
54
  # Make sure that the string '-' is parsed as a nil value.
55
55
  def convert_nillable_string(value, definition)
56
56
  value == '-' ? nil : value
57
57
  end
58
-
58
+
59
59
  # Can be implemented in subclasses for improved categorizations
60
60
  def convert_referer(value, definition)
61
61
  value == '-' ? nil : value
62
62
  end
63
-
63
+
64
64
  # Can be implemented in subclasses for improved categorizations
65
65
  def convert_user_agent(value, definition)
66
66
  value == '-' ? nil : value
67
67
  end
68
68
  end
69
-
69
+
70
70
  end
71
71
  end
@@ -1,6 +1,6 @@
1
1
  module RequestLogAnalyzer::FileFormat
2
-
3
- # The Apache file format is able to log Apache access.log files.
2
+
3
+ # The Apache file format is able to log Apache access.log files.
4
4
  #
5
5
  # The access.log can be configured in Apache to have many different formats. In theory, this
6
6
  # FileFormat can handle any format, but it must be aware of the log formatting that is used
@@ -17,7 +17,10 @@ module RequestLogAnalyzer::FileFormat
17
17
  # A hash of predefined Apache log formats
18
18
  LOG_FORMAT_DEFAULTS = {
19
19
  :common => '%h %l %u %t "%r" %>s %b',
20
- :combined => '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"'
20
+ :combined => '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"',
21
+ :rack => '%h %l %u %t "%r" %>s %b %T',
22
+ :referer => '%{Referer}i -> %U',
23
+ :agent => '%{User-agent}i'
21
24
  }
22
25
 
23
26
  # A hash that defines how the log format directives should be parsed.
@@ -25,18 +28,19 @@ module RequestLogAnalyzer::FileFormat
25
28
  '%' => { :regexp => '%', :captures => [] },
26
29
  'h' => { :regexp => '([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+)', :captures => [{:name => :remote_host, :type => :string}] },
27
30
  'a' => { :regexp => '(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', :captures => [{:name => :remote_ip, :type => :string}] },
28
- 'b' => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :integer}] },
31
+ 'b' => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] },
29
32
  'c' => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] },
30
- 'D' => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :msec}] },
33
+ 'D' => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :musec}] },
31
34
  'l' => { :regexp => '([\w-]+)', :captures => [{:name => :remote_logname, :type => :nillable_string}] },
32
- 'T' => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] },
33
- 't' => { :regexp => '\[([^\]]{26})\]', :captures => [{:name => :timestamp, :type => :timestamp}] },
35
+ 'T' => { :regexp => '((?:\d+(?:\.\d+))|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] },
36
+ 't' => { :regexp => '\[(\d{2}\/[A-Za-z]{3}\/\d{4}.\d{2}:\d{2}:\d{2})(?: .\d{4})?\]', :captures => [{:name => :timestamp, :type => :timestamp}] },
34
37
  's' => { :regexp => '(\d{3})', :captures => [{:name => :http_status, :type => :integer}] },
35
38
  'u' => { :regexp => '(\w+|-)', :captures => [{:name => :user, :type => :nillable_string}] },
36
- 'r' => { :regexp => '([A-Z]+) ([^\s]+) HTTP\/(\d+(?:\.\d+)*)', :captures => [{:name => :http_method, :type => :string},
39
+ 'U' => { :regexp => '(\/\S*)', :captures => [{:name => :path, :type => :string}] },
40
+ 'r' => { :regexp => '([A-Z]+) (\S+) HTTP\/(\d+(?:\.\d+)*)', :captures => [{:name => :http_method, :type => :string},
37
41
  {:name => :path, :type => :path}, {:name => :http_version, :type => :string}]},
38
- 'i' => { 'Referer' => { :regexp => '([^\s]+)', :captures => [{:name => :referer, :type => :nillable_string}] },
39
- 'User-agent' => { :regexp => '(.*)', :captures => [{:name => :user_agent, :type => :user_agent}] }
42
+ 'i' => { 'Referer' => { :regexp => '(\S+)', :captures => [{:name => :referer, :type => :nillable_string}] },
43
+ 'User-agent' => { :regexp => '(.*)', :captures => [{:name => :user_agent, :type => :user_agent}] }
40
44
  }
41
45
  }
42
46
 
@@ -44,8 +48,9 @@ module RequestLogAnalyzer::FileFormat
44
48
  # It will set up the line definition and the report trackers according to the Apache access log format,
45
49
  # which should be passed as first argument. By default, is uses the 'combined' log format.
46
50
  def self.create(*args)
47
- access_line = access_line_definition(args.first)
48
- self.new({ :access => access_line}, report_trackers(access_line))
51
+ access_line = access_line_definition(args.first)
52
+ trackers = report_trackers(access_line) + report_definer.trackers
53
+ self.new(line_definer.line_definitions.merge(:access => access_line), trackers)
49
54
  end
50
55
 
51
56
  # Creates the access log line definition based on the Apache log format string
@@ -58,7 +63,7 @@ module RequestLogAnalyzer::FileFormat
58
63
  format_string.scan(/([^%]*)(?:%(?:\{([^\}]+)\})?>?([A-Za-z%]))?/) do |literal, arg, variable|
59
64
 
60
65
  line_regexp << Regexp.quote(literal) # Make sure to parse the literal before the directive
61
-
66
+
62
67
  if variable
63
68
  # Check if we recognize the log directive
64
69
  directive = LOG_DIRECTIVES[variable]
@@ -73,7 +78,7 @@ module RequestLogAnalyzer::FileFormat
73
78
  end
74
79
  end
75
80
  end
76
-
81
+
77
82
  # Return a new line definition object
78
83
  return RequestLogAnalyzer::LineDefinition.new(:access, :regexp => Regexp.new(line_regexp),
79
84
  :captures => captures, :header => true, :footer => true)
@@ -86,45 +91,47 @@ module RequestLogAnalyzer::FileFormat
86
91
  analyze.timespan if line_definition.captures?(:timestamp)
87
92
  analyze.hourly_spread if line_definition.captures?(:timestamp)
88
93
 
89
- analyze.frequency :category => :http_method, :amount => 20, :title => "HTTP methods" if line_definition.captures?(:http_method)
90
- analyze.frequency :category => :http_status, :amount => 20, :title => "HTTP statuses" if line_definition.captures?(:http_status)
91
- analyze.frequency :category => :path, :amount => 20, :title => "Most popular URIs" if line_definition.captures?(:path)
94
+ analyze.frequency :category => :http_method, :title => "HTTP methods" if line_definition.captures?(:http_method)
95
+ analyze.frequency :category => :http_status, :title => "HTTP statuses" if line_definition.captures?(:http_status)
96
+ analyze.frequency :category => lambda { |r| r.category }, :title => "Most popular URIs" if line_definition.captures?(:path)
92
97
 
93
- analyze.frequency :category => :user_agent, :amount => 20, :title => "User agents" if line_definition.captures?(:user_agent)
94
- analyze.frequency :category => :referer, :amount => 20, :title => "Referers" if line_definition.captures?(:referer)
98
+ analyze.frequency :category => :user_agent, :title => "User agents" if line_definition.captures?(:user_agent)
99
+ analyze.frequency :category => :referer, :title => "Referers" if line_definition.captures?(:referer)
95
100
 
96
- if line_definition.captures?(:path) && line_definition.captures?(:duration)
97
- analyze.duration :duration => :duration, :category => :path , :title => 'Request duration'
98
- end
101
+ analyze.duration :duration => :duration, :category => lambda { |r| r.category }, :title => 'Request duration' if line_definition.captures?(:duration)
102
+ analyze.traffic :traffic => :bytes_sent, :category => lambda { |r| r.category }, :title => 'Traffic' if line_definition.captures?(:bytes_sent)
99
103
 
100
104
  return analyze.trackers
101
105
  end
102
106
 
103
107
  # Define a custom Request class for the Apache file format to speed up timestamp handling.
104
108
  class Request < RequestLogAnalyzer::Request
105
-
109
+
110
+ def category
111
+ first(:path)
112
+ end
113
+
106
114
  MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
107
115
  'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
108
-
116
+
109
117
  # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
110
118
  # to speed up parsing.
111
119
  def convert_timestamp(value, definition)
112
- d = /^(\d{2})\/(\w{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2})/.match(value).captures
113
- "#{d[2]}#{MONTHS[d[1]]}#{d[0]}#{d[3]}#{d[4]}#{d[5]}".to_i
120
+ "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
114
121
  end
115
-
122
+
116
123
  # This function can be overridden to rewrite the path for better categorization in the
117
124
  # reports.
118
125
  def convert_path(value, definition)
119
126
  value
120
127
  end
121
-
122
- # This function can be overridden to simplify the user agent string for better
128
+
129
+ # This function can be overridden to simplify the user agent string for better
123
130
  # categorization in the reports
124
131
  def convert_user_agent(value, definition)
125
132
  value # TODO
126
133
  end
127
-
134
+
128
135
  # Make sure that the string '-' is parsed as a nil value.
129
136
  def convert_nillable_string(value, definition)
130
137
  value == '-' ? nil : value