request-log-analyzer 1.3.7 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. data/LICENSE +3 -3
  2. data/README.rdoc +1 -1
  3. data/bin/request-log-analyzer +17 -14
  4. data/lib/cli/command_line_arguments.rb +51 -51
  5. data/lib/cli/database_console.rb +3 -3
  6. data/lib/cli/database_console_init.rb +2 -2
  7. data/lib/cli/progressbar.rb +10 -10
  8. data/lib/cli/tools.rb +3 -3
  9. data/lib/request_log_analyzer.rb +4 -4
  10. data/lib/request_log_analyzer/aggregator.rb +10 -10
  11. data/lib/request_log_analyzer/aggregator/database_inserter.rb +9 -9
  12. data/lib/request_log_analyzer/aggregator/echo.rb +14 -9
  13. data/lib/request_log_analyzer/aggregator/summarizer.rb +26 -26
  14. data/lib/request_log_analyzer/controller.rb +153 -69
  15. data/lib/request_log_analyzer/database.rb +13 -13
  16. data/lib/request_log_analyzer/database/base.rb +17 -17
  17. data/lib/request_log_analyzer/database/connection.rb +3 -3
  18. data/lib/request_log_analyzer/database/request.rb +2 -2
  19. data/lib/request_log_analyzer/database/source.rb +1 -1
  20. data/lib/request_log_analyzer/file_format.rb +15 -15
  21. data/lib/request_log_analyzer/file_format/amazon_s3.rb +16 -16
  22. data/lib/request_log_analyzer/file_format/apache.rb +20 -19
  23. data/lib/request_log_analyzer/file_format/merb.rb +12 -12
  24. data/lib/request_log_analyzer/file_format/rack.rb +4 -4
  25. data/lib/request_log_analyzer/file_format/rails.rb +146 -70
  26. data/lib/request_log_analyzer/file_format/rails_development.rb +4 -49
  27. data/lib/request_log_analyzer/filter.rb +6 -6
  28. data/lib/request_log_analyzer/filter/anonymize.rb +6 -6
  29. data/lib/request_log_analyzer/filter/field.rb +9 -9
  30. data/lib/request_log_analyzer/filter/timespan.rb +12 -10
  31. data/lib/request_log_analyzer/line_definition.rb +15 -14
  32. data/lib/request_log_analyzer/log_processor.rb +22 -22
  33. data/lib/request_log_analyzer/mailer.rb +15 -9
  34. data/lib/request_log_analyzer/output.rb +53 -12
  35. data/lib/request_log_analyzer/output/fixed_width.rb +40 -41
  36. data/lib/request_log_analyzer/output/html.rb +20 -20
  37. data/lib/request_log_analyzer/request.rb +35 -36
  38. data/lib/request_log_analyzer/source.rb +7 -7
  39. data/lib/request_log_analyzer/source/database_loader.rb +7 -7
  40. data/lib/request_log_analyzer/source/log_parser.rb +48 -43
  41. data/lib/request_log_analyzer/tracker.rb +128 -14
  42. data/lib/request_log_analyzer/tracker/duration.rb +39 -132
  43. data/lib/request_log_analyzer/tracker/frequency.rb +31 -32
  44. data/lib/request_log_analyzer/tracker/hourly_spread.rb +20 -19
  45. data/lib/request_log_analyzer/tracker/timespan.rb +17 -17
  46. data/lib/request_log_analyzer/tracker/traffic.rb +36 -116
  47. data/request-log-analyzer.gemspec +19 -15
  48. data/spec/fixtures/rails_22.log +1 -1
  49. data/spec/integration/command_line_usage_spec.rb +1 -1
  50. data/spec/lib/helpers.rb +7 -7
  51. data/spec/lib/macros.rb +3 -3
  52. data/spec/lib/matchers.rb +41 -27
  53. data/spec/lib/mocks.rb +15 -14
  54. data/spec/lib/testing_format.rb +9 -9
  55. data/spec/spec_helper.rb +6 -6
  56. data/spec/unit/aggregator/database_inserter_spec.rb +13 -13
  57. data/spec/unit/aggregator/summarizer_spec.rb +4 -4
  58. data/spec/unit/controller/controller_spec.rb +2 -2
  59. data/spec/unit/controller/log_processor_spec.rb +1 -1
  60. data/spec/unit/database/base_class_spec.rb +19 -19
  61. data/spec/unit/database/connection_spec.rb +3 -3
  62. data/spec/unit/database/database_spec.rb +25 -25
  63. data/spec/unit/file_format/amazon_s3_format_spec.rb +5 -5
  64. data/spec/unit/file_format/apache_format_spec.rb +13 -13
  65. data/spec/unit/file_format/file_format_api_spec.rb +13 -13
  66. data/spec/unit/file_format/line_definition_spec.rb +24 -17
  67. data/spec/unit/file_format/merb_format_spec.rb +41 -45
  68. data/spec/unit/file_format/rails_format_spec.rb +157 -117
  69. data/spec/unit/filter/anonymize_filter_spec.rb +2 -2
  70. data/spec/unit/filter/field_filter_spec.rb +13 -13
  71. data/spec/unit/filter/filter_spec.rb +1 -1
  72. data/spec/unit/filter/timespan_filter_spec.rb +15 -15
  73. data/spec/unit/mailer_spec.rb +30 -0
  74. data/spec/unit/{source/request_spec.rb → request_spec.rb} +30 -30
  75. data/spec/unit/source/log_parser_spec.rb +27 -27
  76. data/spec/unit/tracker/duration_tracker_spec.rb +115 -78
  77. data/spec/unit/tracker/frequency_tracker_spec.rb +74 -63
  78. data/spec/unit/tracker/hourly_spread_spec.rb +28 -20
  79. data/spec/unit/tracker/timespan_tracker_spec.rb +25 -13
  80. data/spec/unit/tracker/tracker_api_spec.rb +13 -13
  81. data/spec/unit/tracker/traffic_tracker_spec.rb +81 -79
  82. data/tasks/github-gem.rake +125 -75
  83. data/tasks/request_log_analyzer.rake +2 -2
  84. metadata +8 -6
@@ -1,24 +1,24 @@
1
1
  module RequestLogAnalyzer::Filter
2
-
2
+
3
3
  # Filter to select or reject a specific field
4
4
  # Options
5
5
  # * <tt>:mode</tt> :reject or :accept.
6
6
  # * <tt>:field</tt> Specific field to accept or reject.
7
7
  # * <tt>:value</tt> Value that the field should match to be accepted or rejected.
8
8
  class Field < Base
9
-
9
+
10
10
  attr_reader :field, :value, :mode
11
-
11
+
12
12
  def initialize(file_format, options = {})
13
13
  super(file_format, options)
14
14
  setup_filter
15
15
  end
16
-
16
+
17
17
  # Setup mode, field and value.
18
18
  def setup_filter
19
19
  @mode = (@options[:mode] || :accept).to_sym
20
20
  @field = @options[:field].to_sym
21
-
21
+
22
22
  # Convert the timestamp to the correct formats for quick timestamp comparisons
23
23
  if @options[:value].kind_of?(String) && @options[:value][0, 1] == '/' && @options[:value][-1, 1] == '/'
24
24
  @value = Regexp.new(@options[:value][1..-2])
@@ -26,17 +26,17 @@ module RequestLogAnalyzer::Filter
26
26
  @value = @options[:value] # TODO: convert value?
27
27
  end
28
28
  end
29
-
29
+
30
30
  # Keep request if @mode == :select and request has the field and value.
31
31
  # Drop request if @mode == :reject and request has the field and value.
32
32
  # Returns nil otherwise.
33
33
  # <tt>request</tt> Request Object
34
34
  def filter(request)
35
- found_field = request.every(@field).any? { |value| @value === value.to_s }
35
+ found_field = request.every(@field).any? { |value| @value === value.to_s }
36
36
  return nil if !found_field && @mode == :select
37
37
  return nil if found_field && @mode == :reject
38
38
  return request
39
- end
39
+ end
40
40
  end
41
-
41
+
42
42
  end
@@ -1,26 +1,28 @@
1
1
  module RequestLogAnalyzer::Filter
2
-
2
+
3
3
  # Reject all requests not in given timespan
4
4
  # Options
5
5
  # * <tt>:after</tt> Only keep requests after this DateTime.
6
6
  # * <tt>:before</tt> Only keep requests before this DateTime.
7
7
  class Timespan < Base
8
-
8
+
9
9
  attr_reader :before, :after
10
-
10
+
11
11
  def initialize(file_format, options = {})
12
+ @after = nil
13
+ @before = nil
12
14
  super(file_format, options)
13
15
  setup_filter
14
16
  end
15
-
16
-
17
+
18
+
17
19
  # Convert the timestamp to the correct formats for quick timestamp comparisons.
18
20
  # These are stored in the before and after attr_reader fields.
19
21
  def setup_filter
20
- @after = @options[:after].strftime('%Y%m%d%H%M%S').to_i if options[:after]
22
+ @after = @options[:after].strftime('%Y%m%d%H%M%S').to_i if options[:after]
21
23
  @before = @options[:before].strftime('%Y%m%d%H%M%S').to_i if options[:before]
22
24
  end
23
-
25
+
24
26
  # Returns request if:
25
27
  # * @after <= request.timestamp <= @before
26
28
  # * @after <= request.timestamp
@@ -32,12 +34,12 @@ module RequestLogAnalyzer::Filter
32
34
  return request
33
35
  elsif @after && @before.nil? && @after <= request.timestamp
34
36
  return request
35
- elsif @before && @after.nil? && request.timestamp <= @before
37
+ elsif @before && @after.nil? && request.timestamp <= @before
36
38
  return request
37
39
  end
38
40
 
39
41
  return nil
40
- end
42
+ end
41
43
  end
42
-
44
+
43
45
  end
@@ -1,14 +1,14 @@
1
1
  module RequestLogAnalyzer
2
-
2
+
3
3
  # The line definition class is used to specify what lines should be parsed from the log file.
4
4
  # It contains functionality to match a line against the definition and parse the information
5
5
  # from this line. This is used by the LogParser class when parsing a log file..
6
6
  class LineDefinition
7
7
 
8
8
  class Definer
9
-
9
+
10
10
  attr_accessor :line_definitions
11
-
11
+
12
12
  def initialize
13
13
  @line_definitions = {}
14
14
  end
@@ -16,7 +16,7 @@ module RequestLogAnalyzer
16
16
  def initialize_copy(other)
17
17
  @line_definitions = other.line_definitions.dup
18
18
  end
19
-
19
+
20
20
  def method_missing(name, *args, &block)
21
21
  if block_given?
22
22
  @line_definitions[name] = RequestLogAnalyzer::LineDefinition.define(name, &block)
@@ -29,25 +29,26 @@ module RequestLogAnalyzer
29
29
  attr_reader :name
30
30
  attr_accessor :teaser, :regexp, :captures
31
31
  attr_accessor :header, :footer
32
-
32
+
33
33
  alias_method :header?, :header
34
34
  alias_method :footer?, :footer
35
-
35
+
36
36
  # Initializes the LineDefinition instance with a hash containing the different elements of
37
37
  # the definition.
38
38
  def initialize(name, definition = {})
39
39
  @name = name
40
40
  @captures = []
41
+ @teaser = nil
41
42
  definition.each { |key, value| self.send("#{key.to_s}=".to_sym, value) }
42
43
  end
43
-
44
+
44
45
  def self.define(name, &block)
45
46
  definition = self.new(name)
46
47
  yield(definition) if block_given?
47
48
  return definition
48
49
  end
49
-
50
- # Checks whether a given line matches this definition.
50
+
51
+ # Checks whether a given line matches this definition.
51
52
  # It will return false if a line does not match. If the line matches, a hash is returned
52
53
  # with all the fields parsed from that line as content.
53
54
  # If the line definition has a teaser-check, a :teaser_check_failed warning will be emitted
@@ -66,7 +67,7 @@ module RequestLogAnalyzer
66
67
  return false
67
68
  end
68
69
  end
69
-
70
+
70
71
  alias :=~ :matches
71
72
 
72
73
  # matches the line and converts the captured values using the request's
@@ -84,17 +85,17 @@ module RequestLogAnalyzer
84
85
  def convert_captured_values(values, request)
85
86
  value_hash = {}
86
87
  captures.each_with_index do |capture, index|
87
-
88
+
88
89
  # convert the value using the request convert_value function
89
90
  converted = request.convert_value(values[index], capture)
90
91
  value_hash[capture[:name]] ||= converted
91
-
92
+
92
93
  # Add items directly to the resulting hash from the converted value
93
94
  # if it is a hash and they are set in the :provides hash for this line definition
94
95
  if converted.kind_of?(Hash) && capture[:provides].kind_of?(Hash)
95
96
  capture[:provides].each do |name, type|
96
97
  value_hash[name] ||= request.convert_value(converted[name], { :type => type })
97
- end
98
+ end
98
99
  end
99
100
  end
100
101
  return value_hash
@@ -106,5 +107,5 @@ module RequestLogAnalyzer
106
107
  end
107
108
 
108
109
  end
109
-
110
+
110
111
  end
@@ -1,41 +1,41 @@
1
1
  module RequestLogAnalyzer
2
-
2
+
3
3
  # The Logprocessor class is used to perform simple processing actions over log files.
4
- # It will go over the log file/stream line by line, pass the line to a processor and
5
- # write the result back to the output file or stream. The processor can alter the
4
+ # It will go over the log file/stream line by line, pass the line to a processor and
5
+ # write the result back to the output file or stream. The processor can alter the
6
6
  # contents of the line, remain it intact or remove it altogether, based on the current
7
7
  # file format
8
8
  #
9
9
  # Currently, one processors is supported:
10
- # * :strip will remove all irrelevent lines (according to the file format) from the
10
+ # * :strip will remove all irrelevent lines (according to the file format) from the
11
11
  # sources. A compact, information packed log will remain/.
12
12
  #
13
13
  class LogProcessor
14
-
14
+
15
15
  attr_reader :mode, :options, :sources, :file_format
16
16
  attr_accessor :output_file
17
-
17
+
18
18
  # Builds a logprocessor instance from the arguments given on the command line
19
- # <tt>command</tt> The command hat was used to start the log processor. This will set the
19
+ # <tt>command</tt> The command hat was used to start the log processor. This will set the
20
20
  # processing mode. Currently, only :strip is supported.
21
21
  # <tt>arguments</tt> The parsed command line arguments (a CommandLine::Arguments instance)
22
22
  def self.build(command, arguments)
23
-
24
- options = {
25
- :discard_teaser_lines => arguments[:discard_teaser_lines],
26
- :keep_junk_lines => arguments[:keep_junk_lines],
23
+
24
+ options = {
25
+ :discard_teaser_lines => arguments[:discard_teaser_lines],
26
+ :keep_junk_lines => arguments[:keep_junk_lines],
27
27
  }
28
-
28
+
29
29
  log_processor = RequestLogAnalyzer::LogProcessor.new(arguments[:format].to_sym, command, options)
30
30
  log_processor.output_file = arguments[:output] if arguments[:output]
31
31
 
32
32
  arguments.parameters.each do |input|
33
33
  log_processor.sources << input
34
34
  end
35
-
35
+
36
36
  return log_processor
37
37
  end
38
-
38
+
39
39
  # Initializes a new LogProcessor instance.
40
40
  # <tt>format</tt> The file format to use (e.g. :rails).
41
41
  # <tt>mode</tt> The processing mode
@@ -47,7 +47,7 @@ module RequestLogAnalyzer
47
47
  @file_format = format
48
48
  $output_file = nil
49
49
  end
50
-
50
+
51
51
  # Processes input files by opening it and sending the filestream to <code>process_io</code>,
52
52
  # in which the actual processing is performed.
53
53
  # <tt>file</tt> The file to process
@@ -63,7 +63,7 @@ module RequestLogAnalyzer
63
63
  when :strip; io.each_line { |line| @output << strip_line(line) }
64
64
  end
65
65
  end
66
-
66
+
67
67
  # Returns the line itself if the string matches any of the line definitions. If no match is
68
68
  # found, an empty line is returned, which will strip the line from the output.
69
69
  # <tt>line</tt> The line to strip
@@ -72,25 +72,25 @@ module RequestLogAnalyzer
72
72
  end
73
73
 
74
74
  # Runs the log processing by setting up the output stream and iterating over all the
75
- # input sources. Input sources can either be filenames (String instances) or IO streams
75
+ # input sources. Input sources can either be filenames (String instances) or IO streams
76
76
  # (IO instances). The strings "-" and "STDIN" will be substituted for the $stdin variable.
77
77
  def run!
78
78
  if @output_file.nil?
79
- @output = $stdout
79
+ @output = $stdout
80
80
  else
81
- @output = File.new(@output_file, 'a')
81
+ @output = File.new(@output_file, 'a')
82
82
  end
83
-
83
+
84
84
  @sources.each do |source|
85
85
  if source.kind_of?(String) && File.exist?(source)
86
86
  process_file(source)
87
87
  elsif source.kind_of?(IO)
88
88
  process_io(source)
89
89
  elsif ['-', 'STDIN'].include?(source)
90
- process_io($stdin)
90
+ process_io($stdin)
91
91
  end
92
92
  end
93
-
93
+
94
94
  ensure
95
95
  @output.close if @output.kind_of?(File)
96
96
  end
@@ -1,21 +1,23 @@
1
1
  module RequestLogAnalyzer
2
2
 
3
3
  class Mailer
4
-
4
+
5
5
  attr_accessor :data, :to, :host
6
-
6
+
7
7
  # Initialize a mailer
8
8
  # <tt>to</tt> to address
9
9
  # <tt>host</tt> the mailer host
10
10
  # <tt>options</tt> Specific style options
11
+ # Options
12
+ # <tt>:debug</tt> Do not actually mail
11
13
  def initialize(to, host = 'localhost', options = {})
12
- require 'net/smtp'
14
+ require 'net/smtp'
13
15
  @to = to
14
16
  @host = host
15
17
  @options = options
16
18
  @data = []
17
19
  end
18
-
20
+
19
21
  def mail
20
22
  from = @options[:from] || 'contact@railsdoctors.com'
21
23
  from_alias = @options[:from_alias] || 'Request-log-analyzer reporter'
@@ -28,16 +30,20 @@ Subject: #{subject}
28
30
 
29
31
  #{@data.to_s}
30
32
  END_OF_MESSAGE
31
-
32
- Net::SMTP.start(@host) do |smtp|
33
- smtp.send_message msg, from, to
33
+
34
+ unless @options[:debug]
35
+ Net::SMTP.start(@host) do |smtp|
36
+ smtp.send_message msg, from, to
37
+ end
34
38
  end
39
+
40
+ return [msg, from, to]
35
41
  end
36
-
42
+
37
43
  def << string
38
44
  data << string
39
45
  end
40
-
46
+
41
47
  def puts string
42
48
  data << string
43
49
  end
@@ -5,13 +5,49 @@ module RequestLogAnalyzer::Output
5
5
  def self.const_missing(const)
6
6
  RequestLogAnalyzer::load_default_class_file(self, const)
7
7
  end
8
+
9
+ # Loads a Output::Base subclass instance.
10
+ def self.load(file_format, *args)
11
+
12
+ klass = nil
13
+ if file_format.kind_of?(RequestLogAnalyzer::Output::Base)
14
+ # this already is a file format! return itself
15
+ return file_format
16
+
17
+ elsif file_format.kind_of?(Class) && file_format.ancestors.include?(RequestLogAnalyzer::Output::Base)
18
+ # a usable class is provided. Use this format class.
19
+ klass = file_format
20
+
21
+ elsif file_format.kind_of?(String) && File.exist?(file_format)
22
+ # load a format from a ruby file
23
+ require file_format
24
+ const = RequestLogAnalyzer::to_camelcase(File.basename(file_format, '.rb'))
25
+ if RequestLogAnalyzer::FileFormat.const_defined?(const)
26
+ klass = RequestLogAnalyzer::Output.const_get(const)
27
+ elsif Object.const_defined?(const)
28
+ klass = Object.const_get(const)
29
+ else
30
+ raise "Cannot load class #{const} from #{file_format}!"
31
+ end
32
+
33
+ else
34
+ # load a provided file format
35
+ klass = RequestLogAnalyzer::Output.const_get(RequestLogAnalyzer::to_camelcase(file_format))
36
+ end
37
+
38
+ # check the returned klass to see if it can be used
39
+ raise "Could not load a file format from #{file_format.inspect}" if klass.nil?
40
+ raise "Invalid FileFormat class" unless klass.kind_of?(Class) && klass.ancestors.include?(RequestLogAnalyzer::Output::Base)
41
+
42
+ klass.create(*args) # return an instance of the class
43
+ end
8
44
 
9
45
  # Base Class used for generating output for reports.
10
46
  # All output should inherit fromt this class.
11
47
  class Base
12
-
48
+
13
49
  attr_accessor :io, :options, :style
14
-
50
+
15
51
  # Initialize a report
16
52
  # <tt>io</tt> iO Object (file, STDOUT, etc.)
17
53
  # <tt>options</tt> Specific style options
@@ -27,32 +63,37 @@ module RequestLogAnalyzer::Output
27
63
  @style = @style.merge(temp_style)
28
64
  yield(self) if block_given?
29
65
  @style = old_style
30
- end
31
-
66
+ end
67
+
32
68
  # Generate a header for a report
33
69
  def header
34
70
  end
35
-
36
- # Generate the footer of a report
71
+
72
+ # Generate the footer of a report
37
73
  def footer
38
74
  end
39
75
 
76
+ def slice_results(array)
77
+ return array if options[:amount] == :all
78
+ return array.slice(0, options[:amount]) # otherwise
79
+ end
80
+
40
81
  # Generate a report table and push it into the output object.
41
82
  # Yeilds a rows array into which the rows can be pushed
42
83
  # <tt>*colums<tt> Array of Column hashes (see Column options).
43
84
  # <tt>&block</tt>: A block yeilding the rows.
44
- #
85
+ #
45
86
  # === Column options
46
87
  # Columns is an array of hashes containing the column definitions.
47
88
  # * <tt>:align</tt> Alignment :left or :right
48
89
  # * <tt>:treshold</tt> Width in characters or :rest
49
90
  # * <tt>:type</tt> :ratio or nil
50
91
  # * <tt>:width</tt> Width in characters or :rest
51
- #
92
+ #
52
93
  # === Example
53
94
  # The output object should support table definitions:
54
95
  #
55
- # output.table({:align => :left}, {:align => :right }, {:align => :right}, {:type => :ratio, :width => :rest}) do |rows|
96
+ # output.table({:align => :left}, {:align => :right }, {:align => :right}, {:type => :ratio, :width => :rest}) do |rows|
56
97
  # sorted_frequencies.each do |(cat, count)|
57
98
  # rows << [cat, "#{count} hits", '%0.1f%%' % ((count.to_f / total_hits.to_f) * 100.0), (count.to_f / total_hits.to_f)]
58
99
  # end
@@ -60,13 +101,13 @@ module RequestLogAnalyzer::Output
60
101
  #
61
102
  def table(*columns, &block)
62
103
  end
63
-
104
+
64
105
  protected
65
106
  # Check if a given table defination hash includes a header (title)
66
107
  # <tt>columns</tt> The columns hash
67
108
  def table_has_header?(columns)
68
- columns.any? { |column| !column[:title].nil? }
109
+ columns.any? { |column| !column[:title].nil? }
69
110
  end
70
-
111
+
71
112
  end
72
113
  end