request-log-analyzer 1.3.7 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. data/LICENSE +3 -3
  2. data/README.rdoc +1 -1
  3. data/bin/request-log-analyzer +17 -14
  4. data/lib/cli/command_line_arguments.rb +51 -51
  5. data/lib/cli/database_console.rb +3 -3
  6. data/lib/cli/database_console_init.rb +2 -2
  7. data/lib/cli/progressbar.rb +10 -10
  8. data/lib/cli/tools.rb +3 -3
  9. data/lib/request_log_analyzer.rb +4 -4
  10. data/lib/request_log_analyzer/aggregator.rb +10 -10
  11. data/lib/request_log_analyzer/aggregator/database_inserter.rb +9 -9
  12. data/lib/request_log_analyzer/aggregator/echo.rb +14 -9
  13. data/lib/request_log_analyzer/aggregator/summarizer.rb +26 -26
  14. data/lib/request_log_analyzer/controller.rb +153 -69
  15. data/lib/request_log_analyzer/database.rb +13 -13
  16. data/lib/request_log_analyzer/database/base.rb +17 -17
  17. data/lib/request_log_analyzer/database/connection.rb +3 -3
  18. data/lib/request_log_analyzer/database/request.rb +2 -2
  19. data/lib/request_log_analyzer/database/source.rb +1 -1
  20. data/lib/request_log_analyzer/file_format.rb +15 -15
  21. data/lib/request_log_analyzer/file_format/amazon_s3.rb +16 -16
  22. data/lib/request_log_analyzer/file_format/apache.rb +20 -19
  23. data/lib/request_log_analyzer/file_format/merb.rb +12 -12
  24. data/lib/request_log_analyzer/file_format/rack.rb +4 -4
  25. data/lib/request_log_analyzer/file_format/rails.rb +146 -70
  26. data/lib/request_log_analyzer/file_format/rails_development.rb +4 -49
  27. data/lib/request_log_analyzer/filter.rb +6 -6
  28. data/lib/request_log_analyzer/filter/anonymize.rb +6 -6
  29. data/lib/request_log_analyzer/filter/field.rb +9 -9
  30. data/lib/request_log_analyzer/filter/timespan.rb +12 -10
  31. data/lib/request_log_analyzer/line_definition.rb +15 -14
  32. data/lib/request_log_analyzer/log_processor.rb +22 -22
  33. data/lib/request_log_analyzer/mailer.rb +15 -9
  34. data/lib/request_log_analyzer/output.rb +53 -12
  35. data/lib/request_log_analyzer/output/fixed_width.rb +40 -41
  36. data/lib/request_log_analyzer/output/html.rb +20 -20
  37. data/lib/request_log_analyzer/request.rb +35 -36
  38. data/lib/request_log_analyzer/source.rb +7 -7
  39. data/lib/request_log_analyzer/source/database_loader.rb +7 -7
  40. data/lib/request_log_analyzer/source/log_parser.rb +48 -43
  41. data/lib/request_log_analyzer/tracker.rb +128 -14
  42. data/lib/request_log_analyzer/tracker/duration.rb +39 -132
  43. data/lib/request_log_analyzer/tracker/frequency.rb +31 -32
  44. data/lib/request_log_analyzer/tracker/hourly_spread.rb +20 -19
  45. data/lib/request_log_analyzer/tracker/timespan.rb +17 -17
  46. data/lib/request_log_analyzer/tracker/traffic.rb +36 -116
  47. data/request-log-analyzer.gemspec +19 -15
  48. data/spec/fixtures/rails_22.log +1 -1
  49. data/spec/integration/command_line_usage_spec.rb +1 -1
  50. data/spec/lib/helpers.rb +7 -7
  51. data/spec/lib/macros.rb +3 -3
  52. data/spec/lib/matchers.rb +41 -27
  53. data/spec/lib/mocks.rb +15 -14
  54. data/spec/lib/testing_format.rb +9 -9
  55. data/spec/spec_helper.rb +6 -6
  56. data/spec/unit/aggregator/database_inserter_spec.rb +13 -13
  57. data/spec/unit/aggregator/summarizer_spec.rb +4 -4
  58. data/spec/unit/controller/controller_spec.rb +2 -2
  59. data/spec/unit/controller/log_processor_spec.rb +1 -1
  60. data/spec/unit/database/base_class_spec.rb +19 -19
  61. data/spec/unit/database/connection_spec.rb +3 -3
  62. data/spec/unit/database/database_spec.rb +25 -25
  63. data/spec/unit/file_format/amazon_s3_format_spec.rb +5 -5
  64. data/spec/unit/file_format/apache_format_spec.rb +13 -13
  65. data/spec/unit/file_format/file_format_api_spec.rb +13 -13
  66. data/spec/unit/file_format/line_definition_spec.rb +24 -17
  67. data/spec/unit/file_format/merb_format_spec.rb +41 -45
  68. data/spec/unit/file_format/rails_format_spec.rb +157 -117
  69. data/spec/unit/filter/anonymize_filter_spec.rb +2 -2
  70. data/spec/unit/filter/field_filter_spec.rb +13 -13
  71. data/spec/unit/filter/filter_spec.rb +1 -1
  72. data/spec/unit/filter/timespan_filter_spec.rb +15 -15
  73. data/spec/unit/mailer_spec.rb +30 -0
  74. data/spec/unit/{source/request_spec.rb → request_spec.rb} +30 -30
  75. data/spec/unit/source/log_parser_spec.rb +27 -27
  76. data/spec/unit/tracker/duration_tracker_spec.rb +115 -78
  77. data/spec/unit/tracker/frequency_tracker_spec.rb +74 -63
  78. data/spec/unit/tracker/hourly_spread_spec.rb +28 -20
  79. data/spec/unit/tracker/timespan_tracker_spec.rb +25 -13
  80. data/spec/unit/tracker/tracker_api_spec.rb +13 -13
  81. data/spec/unit/tracker/traffic_tracker_spec.rb +81 -79
  82. data/tasks/github-gem.rake +125 -75
  83. data/tasks/request_log_analyzer.rake +2 -2
  84. metadata +8 -6
@@ -5,13 +5,13 @@ class RequestLogAnalyzer::Database
5
5
 
6
6
  def self.const_missing(const) # :nodoc:
7
7
  RequestLogAnalyzer::load_default_class_file(self, const)
8
- end
8
+ end
9
9
 
10
10
  include RequestLogAnalyzer::Database::Connection
11
11
 
12
12
  attr_accessor :file_format
13
13
  attr_reader :line_classes
14
-
14
+
15
15
  def initialize(connection_identifier = nil)
16
16
  @line_classes = []
17
17
  RequestLogAnalyzer::Database::Base.database = self
@@ -23,11 +23,11 @@ class RequestLogAnalyzer::Database
23
23
  line_type = line_type.name if line_type.respond_to?(:name)
24
24
  Object.const_get("#{line_type}_line".camelize)
25
25
  end
26
-
26
+
27
27
  def default_classes
28
28
  [RequestLogAnalyzer::Database::Request, RequestLogAnalyzer::Database::Source, RequestLogAnalyzer::Database::Warning]
29
29
  end
30
-
30
+
31
31
  # Loads the ORM classes by inspecting the tables in the current database
32
32
  def load_database_schema!
33
33
  connection.tables.map do |table|
@@ -39,16 +39,16 @@ class RequestLogAnalyzer::Database
39
39
  end
40
40
  end
41
41
  end
42
-
42
+
43
43
  # Returns an array of all the ActiveRecord-bases ORM classes for this database
44
44
  def orm_classes
45
45
  default_classes + line_classes
46
46
  end
47
-
47
+
48
48
  # Loads an ActiveRecord-based class that correspond to the given parameter, which can either be
49
49
  # a table name or a LineDefinition instance.
50
50
  def load_activerecord_class(linedefinition_or_table)
51
-
51
+
52
52
  case linedefinition_or_table
53
53
  when String, Symbol
54
54
  klass_name = linedefinition_or_table.to_s.singularize.camelize
@@ -57,12 +57,12 @@ class RequestLogAnalyzer::Database
57
57
  klass_name = "#{linedefinition_or_table.name}_line".camelize
58
58
  klass = RequestLogAnalyzer::Database::Base.subclass_from_line_definition(linedefinition_or_table)
59
59
  end
60
-
60
+
61
61
  Object.const_set(klass_name, klass)
62
62
  klass = Object.const_get(klass_name)
63
63
  @line_classes << klass
64
64
  return klass
65
- end
65
+ end
66
66
 
67
67
  def fileformat_classes
68
68
  raise "No file_format provided!" unless file_format
@@ -70,25 +70,25 @@ class RequestLogAnalyzer::Database
70
70
  return default_classes + line_classes
71
71
  end
72
72
 
73
- # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
73
+ # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
74
74
  # file format definition. These ORM classes will later be used to create records in the database.
75
75
  def create_database_schema!
76
76
  fileformat_classes.each { |klass| klass.create_table! }
77
77
  end
78
-
78
+
79
79
  # Drops the table of all the ORM classes, and unregisters the classes
80
80
  def drop_database_schema!
81
81
  file_format ? fileformat_classes.map(&:drop_table!) : orm_classes.map(&:drop_table!)
82
82
  remove_orm_classes!
83
83
  end
84
-
84
+
85
85
  # Registers the default ORM classes in the default namespace
86
86
  def register_default_orm_classes!
87
87
  Object.const_set('Request', RequestLogAnalyzer::Database::Request)
88
88
  Object.const_set('Source', RequestLogAnalyzer::Database::Source)
89
89
  Object.const_set('Warning', RequestLogAnalyzer::Database::Warning)
90
90
  end
91
-
91
+
92
92
  # Unregisters every ORM class constant
93
93
  def remove_orm_classes!
94
94
  orm_classes.each do |klass|
@@ -1,5 +1,5 @@
1
1
  class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
2
-
2
+
3
3
  self.abstract_class = true
4
4
 
5
5
  def <=>(other)
@@ -13,20 +13,20 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
13
13
  def line_type
14
14
  self.class.name.underscore.gsub(/_line$/, '').to_sym
15
15
  end
16
-
16
+
17
17
  class_inheritable_accessor :line_definition
18
18
  cattr_accessor :database
19
19
 
20
20
  def self.subclass_from_line_definition(definition)
21
21
  klass = Class.new(RequestLogAnalyzer::Database::Base)
22
22
  klass.set_table_name("#{definition.name}_lines")
23
-
23
+
24
24
  klass.line_definition = definition
25
-
25
+
26
26
  # Set relations with requests and sources table
27
27
  klass.belongs_to :request
28
28
  klass.belongs_to :source
29
-
29
+
30
30
  # Serialize complex fields into the database
31
31
  definition.captures.select { |c| c.has_key?(:provides) }.each do |capture|
32
32
  klass.send(:serialize, capture[:name], Hash)
@@ -48,32 +48,32 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
48
48
  klass.belongs_to :request
49
49
  RequestLogAnalyzer::Database::Request.has_many table.to_sym
50
50
  end
51
-
51
+
52
52
  if klass.column_names.include?('source_id')
53
53
  klass.belongs_to :source
54
54
  RequestLogAnalyzer::Database::Source.has_many table.to_sym
55
55
  end
56
-
56
+
57
57
  return klass
58
58
  end
59
-
59
+
60
60
  def self.drop_table!
61
61
  database.connection.remove_index(self.table_name, [:source_id]) rescue nil
62
62
  database.connection.remove_index(self.table_name, [:request_id]) rescue nil
63
63
  database.connection.drop_table(self.table_name) if database.connection.table_exists?(self.table_name)
64
64
  end
65
-
65
+
66
66
  def self.create_table!
67
67
  raise "No line_definition available to base table schema on!" unless self.line_definition
68
-
68
+
69
69
  unless table_exists?
70
70
  database.connection.create_table(table_name.to_sym) do |t|
71
-
71
+
72
72
  # Default fields
73
73
  t.column :request_id, :integer
74
74
  t.column :source_id, :integer
75
75
  t.column :lineno, :integer
76
-
76
+
77
77
  line_definition.captures.each do |capture|
78
78
  # Add a field for every capture
79
79
  t.column(capture[:name], column_type(capture[:type]))
@@ -83,13 +83,13 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
83
83
  end
84
84
  end
85
85
  end
86
-
86
+
87
87
  # Add indices to table for more speedy querying
88
88
  database.connection.add_index(self.table_name.to_sym, [:request_id]) # rescue
89
89
  database.connection.add_index(self.table_name.to_sym, [:source_id]) # rescue
90
90
  end
91
-
92
-
91
+
92
+
93
93
  # Function to determine the column type for a field
94
94
  # TODO: make more robust / include in file-format definition
95
95
  def self.column_type(type_indicator)
@@ -110,6 +110,6 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
110
110
  when :date; :date
111
111
  else :string
112
112
  end
113
- end
114
-
113
+ end
114
+
115
115
  end
@@ -26,13 +26,13 @@ module RequestLogAnalyzer::Database::Connection
26
26
  raise "Cannot connect with this connection_identifier: #{connection_identifier.inspect}"
27
27
  end
28
28
  end
29
-
29
+
30
30
  def disconnect
31
31
  RequestLogAnalyzer::Database::Base.remove_connection
32
32
  end
33
-
33
+
34
34
  def connection
35
35
  RequestLogAnalyzer::Database::Base.connection
36
36
  end
37
-
37
+
38
38
  end
@@ -1,5 +1,5 @@
1
1
  class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base
2
-
2
+
3
3
  # Returns an array of all the Line objects of this request in the correct order.
4
4
  def lines
5
5
  @lines ||= begin
@@ -8,7 +8,7 @@ class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base
8
8
  lines.sort
9
9
  end
10
10
  end
11
-
11
+
12
12
  # Creates the table to store requests in.
13
13
  def self.create_table!
14
14
  unless database.connection.table_exists?(:requests)
@@ -1,5 +1,5 @@
1
1
  class RequestLogAnalyzer::Database::Source < RequestLogAnalyzer::Database::Base
2
-
2
+
3
3
  def self.create_table!
4
4
  unless database.connection.table_exists?(:sources)
5
5
  database.connection.create_table(:sources) do |t|
@@ -1,9 +1,9 @@
1
1
  module RequestLogAnalyzer::FileFormat
2
-
2
+
3
3
  def self.const_missing(const)
4
4
  RequestLogAnalyzer::load_default_class_file(self, const)
5
- end
6
-
5
+ end
6
+
7
7
  # Loads a FileFormat::Base subclass instance.
8
8
  # You can provide:
9
9
  # * A FileFormat instance (which will return itself)
@@ -34,46 +34,46 @@ module RequestLogAnalyzer::FileFormat
34
34
 
35
35
  else
36
36
  # load a provided file format
37
- klass = RequestLogAnalyzer::FileFormat.const_get(RequestLogAnalyzer::to_camelcase(file_format))
37
+ klass = RequestLogAnalyzer::FileFormat.const_get(RequestLogAnalyzer::to_camelcase(file_format))
38
38
  end
39
-
39
+
40
40
  # check the returned klass to see if it can be used
41
41
  raise "Could not load a file format from #{file_format.inspect}" if klass.nil?
42
42
  raise "Invalid FileFormat class" unless klass.kind_of?(Class) && klass.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
43
-
43
+
44
44
  @current_file_format = klass.create(*args) # return an instance of the class
45
45
  end
46
46
 
47
- # Base class for all log file format definitions. This class provides functions for subclasses to
47
+ # Base class for all log file format definitions. This class provides functions for subclasses to
48
48
  # define their LineDefinitions and to define a summary report.
49
49
  #
50
50
  # A subclass of this class is instantiated when request-log-analyzer is started and this instance
51
51
  # is shared with all components of the application so they can act on the specifics of the format
52
52
  class Base
53
-
53
+
54
54
  attr_reader :line_definitions, :report_trackers
55
-
55
+
56
56
  ####################################################################################
57
57
  # CLASS METHODS for format definition
58
58
  ####################################################################################
59
-
59
+
60
60
  # Registers the line definer instance for a subclass.
61
61
  def self.inherited(subclass)
62
62
  if subclass.superclass == RequestLogAnalyzer::FileFormat::Base
63
63
 
64
64
  # Create aline and report definer for this class
65
- subclass.class_eval do
65
+ subclass.class_eval do
66
66
  instance_variable_set(:@line_definer, RequestLogAnalyzer::LineDefinition::Definer.new)
67
67
  instance_variable_set(:@report_definer, RequestLogAnalyzer::Aggregator::Summarizer::Definer.new)
68
68
  class << self; attr_accessor :line_definer, :report_definer; end
69
- end
69
+ end
70
70
 
71
71
  # Create a custom Request class for this file format
72
72
  subclass.const_set('Request', Class.new(RequestLogAnalyzer::Request)) unless subclass.const_defined?('Request')
73
73
  else
74
74
 
75
75
  # Copy the line and report definer from the parent class.
76
- subclass.class_eval do
76
+ subclass.class_eval do
77
77
  instance_variable_set(:@line_definer, superclass.line_definer.clone)
78
78
  instance_variable_set(:@report_definer, superclass.report_definer.clone)
79
79
  class << self; attr_accessor :line_definer, :report_definer; end
@@ -113,7 +113,7 @@ module RequestLogAnalyzer::FileFormat
113
113
  return self.new(line_definer.line_definitions, report_definer.trackers)
114
114
  end
115
115
 
116
- def initialize(line_definitions = [], report_trackers = [])
116
+ def initialize(line_definitions = {}, report_trackers = [])
117
117
  @line_definitions, @report_trackers = line_definitions, report_trackers
118
118
  end
119
119
 
@@ -153,7 +153,7 @@ module RequestLogAnalyzer::FileFormat
153
153
  match = definition.matches(line, &warning_handler)
154
154
  return match if match
155
155
  end
156
-
156
+
157
157
  return nil
158
158
  end
159
159
  end
@@ -1,20 +1,20 @@
1
1
  module RequestLogAnalyzer::FileFormat
2
2
 
3
- # FileFormat for Amazon S3 access logs.
3
+ # FileFormat for Amazon S3 access logs.
4
4
  #
5
5
  # Access logs are disabled by default on Amazon S3. To enable logging, see
6
6
  # http://docs.amazonwebservices.com/AmazonS3/latest/index.html?ServerLogs.html
7
7
  class AmazonS3 < Base
8
-
8
+
9
9
  line_definition :access do |line|
10
10
  line.header = true
11
11
  line.footer = true
12
12
  line.regexp = /^([^\ ]+) ([^\ ]+) \[(\d{2}\/[A-Za-z]{3}\/\d{4}.\d{2}:\d{2}:\d{2})(?: .\d{4})?\] (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]+)" "([^"]+)"/
13
- line.captures << { :name => :bucket_owner, :type => :string } <<
14
- { :name => :bucket, :type => :string } <<
13
+ line.captures << { :name => :bucket_owner, :type => :string } <<
14
+ { :name => :bucket, :type => :string } <<
15
15
  { :name => :timestamp, :type => :timestamp } <<
16
16
  { :name => :remote_ip, :type => :string } <<
17
- { :name => :requester, :type => :string } <<
17
+ { :name => :requester, :type => :string } <<
18
18
  { :name => :request_id, :type => :string } <<
19
19
  { :name => :operation, :type => :string } <<
20
20
  { :name => :key, :type => :nillable_string } <<
@@ -28,44 +28,44 @@ module RequestLogAnalyzer::FileFormat
28
28
  { :name => :referer, :type => :referer } <<
29
29
  { :name => :user_agent, :type => :user_agent }
30
30
  end
31
-
31
+
32
32
  report do |analyze|
33
33
  analyze.timespan
34
34
  analyze.hourly_spread
35
35
 
36
- analyze.frequency :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :amount => 20, :title => "Most popular items"
37
- analyze.duration :duration => :total_time, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :amount => 20, :title => "Request duration"
38
- analyze.traffic :traffic => :bytes_sent, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :amount => 20, :title => "Traffic"
36
+ analyze.frequency :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Most popular items"
37
+ analyze.duration :duration => :total_time, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Request duration"
38
+ analyze.traffic :traffic => :bytes_sent, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Traffic"
39
39
  analyze.frequency :category => :http_status, :title => 'HTTP status codes'
40
40
  analyze.frequency :category => :error_code, :title => 'Error codes'
41
41
  end
42
-
42
+
43
43
  class Request < RequestLogAnalyzer::Request
44
-
44
+
45
45
  MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
46
46
  'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
47
-
47
+
48
48
  # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
49
49
  # to speed up parsing.
50
50
  def convert_timestamp(value, definition)
51
51
  "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
52
52
  end
53
-
53
+
54
54
  # Make sure that the string '-' is parsed as a nil value.
55
55
  def convert_nillable_string(value, definition)
56
56
  value == '-' ? nil : value
57
57
  end
58
-
58
+
59
59
  # Can be implemented in subclasses for improved categorizations
60
60
  def convert_referer(value, definition)
61
61
  value == '-' ? nil : value
62
62
  end
63
-
63
+
64
64
  # Can be implemented in subclasses for improved categorizations
65
65
  def convert_user_agent(value, definition)
66
66
  value == '-' ? nil : value
67
67
  end
68
68
  end
69
-
69
+
70
70
  end
71
71
  end
@@ -1,6 +1,6 @@
1
1
  module RequestLogAnalyzer::FileFormat
2
-
3
- # The Apache file format is able to log Apache access.log files.
2
+
3
+ # The Apache file format is able to log Apache access.log files.
4
4
  #
5
5
  # The access.log can be configured in Apache to have many different formats. In theory, this
6
6
  # FileFormat can handle any format, but it must be aware of the log formatting that is used
@@ -22,7 +22,7 @@ module RequestLogAnalyzer::FileFormat
22
22
  :referer => '%{Referer}i -> %U',
23
23
  :agent => '%{User-agent}i'
24
24
  }
25
-
25
+
26
26
  # A hash that defines how the log format directives should be parsed.
27
27
  LOG_DIRECTIVES = {
28
28
  '%' => { :regexp => '%', :captures => [] },
@@ -48,8 +48,9 @@ module RequestLogAnalyzer::FileFormat
48
48
  # It will set up the line definition and the report trackers according to the Apache access log format,
49
49
  # which should be passed as first argument. By default, is uses the 'combined' log format.
50
50
  def self.create(*args)
51
- access_line = access_line_definition(args.first)
52
- self.new({ :access => access_line}, report_trackers(access_line))
51
+ access_line = access_line_definition(args.first)
52
+ trackers = report_trackers(access_line) + report_definer.trackers
53
+ self.new(line_definer.line_definitions.merge(:access => access_line), trackers)
53
54
  end
54
55
 
55
56
  # Creates the access log line definition based on the Apache log format string
@@ -62,7 +63,7 @@ module RequestLogAnalyzer::FileFormat
62
63
  format_string.scan(/([^%]*)(?:%(?:\{([^\}]+)\})?>?([A-Za-z%]))?/) do |literal, arg, variable|
63
64
 
64
65
  line_regexp << Regexp.quote(literal) # Make sure to parse the literal before the directive
65
-
66
+
66
67
  if variable
67
68
  # Check if we recognize the log directive
68
69
  directive = LOG_DIRECTIVES[variable]
@@ -77,7 +78,7 @@ module RequestLogAnalyzer::FileFormat
77
78
  end
78
79
  end
79
80
  end
80
-
81
+
81
82
  # Return a new line definition object
82
83
  return RequestLogAnalyzer::LineDefinition.new(:access, :regexp => Regexp.new(line_regexp),
83
84
  :captures => captures, :header => true, :footer => true)
@@ -90,12 +91,12 @@ module RequestLogAnalyzer::FileFormat
90
91
  analyze.timespan if line_definition.captures?(:timestamp)
91
92
  analyze.hourly_spread if line_definition.captures?(:timestamp)
92
93
 
93
- analyze.frequency :category => :http_method, :amount => 20, :title => "HTTP methods" if line_definition.captures?(:http_method)
94
- analyze.frequency :category => :http_status, :amount => 20, :title => "HTTP statuses" if line_definition.captures?(:http_status)
95
- analyze.frequency :category => lambda { |r| r.category }, :amount => 20, :title => "Most popular URIs" if line_definition.captures?(:path)
94
+ analyze.frequency :category => :http_method, :title => "HTTP methods" if line_definition.captures?(:http_method)
95
+ analyze.frequency :category => :http_status, :title => "HTTP statuses" if line_definition.captures?(:http_status)
96
+ analyze.frequency :category => lambda { |r| r.category }, :title => "Most popular URIs" if line_definition.captures?(:path)
96
97
 
97
- analyze.frequency :category => :user_agent, :amount => 20, :title => "User agents" if line_definition.captures?(:user_agent)
98
- analyze.frequency :category => :referer, :amount => 20, :title => "Referers" if line_definition.captures?(:referer)
98
+ analyze.frequency :category => :user_agent, :title => "User agents" if line_definition.captures?(:user_agent)
99
+ analyze.frequency :category => :referer, :title => "Referers" if line_definition.captures?(:referer)
99
100
 
100
101
  analyze.duration :duration => :duration, :category => lambda { |r| r.category }, :title => 'Request duration' if line_definition.captures?(:duration)
101
102
  analyze.traffic :traffic => :bytes_sent, :category => lambda { |r| r.category }, :title => 'Traffic' if line_definition.captures?(:bytes_sent)
@@ -105,32 +106,32 @@ module RequestLogAnalyzer::FileFormat
105
106
 
106
107
  # Define a custom Request class for the Apache file format to speed up timestamp handling.
107
108
  class Request < RequestLogAnalyzer::Request
108
-
109
+
109
110
  def category
110
111
  first(:path)
111
112
  end
112
-
113
+
113
114
  MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
114
115
  'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
115
-
116
+
116
117
  # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
117
118
  # to speed up parsing.
118
119
  def convert_timestamp(value, definition)
119
120
  "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
120
121
  end
121
-
122
+
122
123
  # This function can be overridden to rewrite the path for better categorization in the
123
124
  # reports.
124
125
  def convert_path(value, definition)
125
126
  value
126
127
  end
127
-
128
- # This function can be overridden to simplify the user agent string for better
128
+
129
+ # This function can be overridden to simplify the user agent string for better
129
130
  # categorization in the reports
130
131
  def convert_user_agent(value, definition)
131
132
  value # TODO
132
133
  end
133
-
134
+
134
135
  # Make sure that the string '-' is parsed as a nil value.
135
136
  def convert_nillable_string(value, definition)
136
137
  value == '-' ? nil : value